diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml new file mode 100644 index 00000000..fdbc71c9 --- /dev/null +++ b/.github/actions/setup-sqlite-version/action.yml @@ -0,0 +1,39 @@ +name: "Setup SQLite version" +description: "Build and activate a specific SQLite version from its amalgamation archive" +inputs: + version: + description: "The SQLite version to install" + required: true + cflags: + description: "CFLAGS to use when compiling SQLite" + required: false + default: "" + skip-activate: + description: "Set to true to skip modifying the library path" + required: false + default: "false" + fallback-urls: + description: "Whitespace-separated fallback download URLs to try after sqlite.org" + required: false + default: "" +outputs: + sqlite-location: + description: "Directory containing the compiled SQLite library" + value: ${{ steps.build.outputs.sqlite-location }} +runs: + using: "composite" + steps: + - shell: bash + run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/sqlite-versions/downloads + key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 + - id: build + shell: bash + run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" + env: + SQLITE_VERSION: ${{ inputs.version }} + SQLITE_CFLAGS: ${{ inputs.cflags }} + SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} + SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh new file mode 100644 index 00000000..03d6a68f --- /dev/null +++ b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" +cflags="${SQLITE_CFLAGS:-}" +skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" +extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" + +case "$version_spec" in + 3.46 | 3.46.0) + sqlite_version="3.46.0" + sqlite_year="2024" + amalgamation_id="3460000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" + ;; + 3.25 | 3.25.0) + sqlite_version="3.25.0" + sqlite_year="2018" + amalgamation_id="3250000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1" + ;; + *) + echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." + exit 1 + ;; +esac + +case "$(uname -s)" in + Linux) + library_name="libsqlite3.so.0" + library_path_var="LD_LIBRARY_PATH" + ;; + Darwin) + library_name="libsqlite3.dylib" + library_path_var="DYLD_LIBRARY_PATH" + ;; + *) + echo "::error::Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +runner_temp="${RUNNER_TEMP:-}" +if [ -z "$runner_temp" ]; then + runner_temp="$(mktemp -d)" +fi + +filename="sqlite-amalgamation-${amalgamation_id}" +official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" +download_dir="${runner_temp}/sqlite-versions/downloads" +source_root="${runner_temp}/sqlite-versions/source" +source_dir="${source_root}/${filename}" +build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" +archive_path="${download_dir}/${filename}.zip" + +mkdir -p "$download_dir" "$source_root" "$build_dir" + +download_archive() { + local url + local candidate_path="${archive_path}.tmp" + local urls=("$official_url") + + for url in $builtin_fallback_urls $extra_fallback_urls; do + urls+=("$url") + done + + rm -f "$candidate_path" + for url in "${urls[@]}"; do + echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" + if curl \ + --fail \ + --location \ + --show-error \ + --retry 5 \ + --retry-delay 2 \ + --retry-max-time 180 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 240 \ + --output "$candidate_path" \ + "$url"; then + mv "$candidate_path" "$archive_path" + return 0 + fi + + echo "::warning::Download failed from ${url}" + rm -f "$candidate_path" + done + + echo "::error::Could not download SQLite ${sqlite_version} amalgamation" + return 1 +} + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + if [ ! -f "$archive_path" ]; then + download_archive + fi + + rm -rf "$source_dir" + unzip -q "$archive_path" -d "$source_root" +fi + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" + exit 1 +fi + +read -r -a cflag_args <<< "$cflags" + +echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" +gcc \ + -fPIC \ + -shared \ + "${cflag_args[@]}" \ + "${source_dir}/sqlite3.c" \ + "-I${source_dir}" \ + -o "${build_dir}/${library_name}" + +if [ "$library_name" = "libsqlite3.so.0" ]; then + ln -sf "$library_name" "${build_dir}/libsqlite3.so" +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" +else + echo "sqlite-location=${build_dir}" +fi + +case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in + true | 1 | yes) + echo "Skipping ${library_path_var} activation" + ;; + *) + existing_value="${!library_path_var:-}" + if [ -n "${GITHUB_ENV:-}" ]; then + if [ -n "$existing_value" ]; then + echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" + else + echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" + fi + fi + echo "Added ${build_dir} to ${library_path_var}" + ;; +esac diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b0640ae8..cf9b25a7 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,24 +14,46 @@ jobs: deploy: runs-on: ubuntu-latest steps: + - name: Check deployment prerequisites + id: deployment-prerequisites + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} + run: | + missing=() + for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do + if [[ -z "${!variable:-}" ]]; then + missing+=("$variable") + fi + done + if (( ${#missing[@]} )); then + echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi - name: Check out datasette - uses: actions/checkout@v6 + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} + uses: actions/checkout@v7 - name: Set up Python + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Run tests - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -39,14 +61,18 @@ jobs: fixtures-metadata.json \ plugins \ --extra-db-filename extra_database.db + # Package the config with the plugins, excluding test-only plugin secrets + # that reference temporary files outside the deployed container. + jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json - name: Build docs.db - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -58,6 +84,7 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Deploy to docs as well (only for main) - if: ${{ github.ref == 'refs/heads/main' }} + - name: Upload latest documentation database to S3 (only for main) + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - # Deploy docs.db to a different service - datasette publish cloudrun docs.db \ - --branch=$GITHUB_SHA \ - --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1" \ - --service=datasette-docs-latest + # Keep development documentation separate from the stable release database. + s3-credentials put-object datasette-docs latest/docs.db docs.db \ + --content-type application/octet-stream diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index b8fb8aaa..00000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Read the Docs Pull Request Preview -on: - pull_request: - types: - - opened - -permissions: - pull-requests: write - -jobs: - documentation-links: - runs-on: ubuntu-latest - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "datasette" diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 5275ddef..85369f6c 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,9 +2,15 @@ name: Playwright on: push: + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read @@ -16,7 +22,7 @@ jobs: matrix: browser: [chromium, firefox, webkit] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python 3.14 uses: actions/setup-python@v6 with: @@ -25,14 +31,14 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Cache uv - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/uv key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} restore-keys: | ${{ runner.os }}-py3.14-uv- - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 735e14e9..fa7ec6aa 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -1,6 +1,15 @@ name: Check JavaScript for conformance with Prettier -on: [push] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -10,8 +19,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v6 - - uses: actions/cache@v5 + uses: actions/checkout@v7 + - uses: actions/cache@v6 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87300593..232a34c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [created] + types: [published] permissions: contents: read @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: @@ -51,12 +51,14 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 + # After the first non-prerelease 1.0 release, disable this job on 0.65.x, + # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: @@ -66,33 +68,27 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy stable-docs.datasette.io to Cloud Run + - name: Upload stable documentation database to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - datasette publish cloudrun docs.db \ - --service=datasette-docs-stable + s3-credentials put-object datasette-docs docs.db docs.db \ + --content-type application/octet-stream deploy_docker: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index e622ef4c..c5a4f0db 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 9a808194..aa35338f 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -1,6 +1,15 @@ name: Check spelling in documentation -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -9,7 +18,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 59b5fbc0..ecde5940 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml deleted file mode 100644 index c514048e..00000000 --- a/.github/workflows/test-coverage.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Calculate test coverage - -on: - push: - branches: - - main - pull_request: - branches: - - main -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Check out datasette - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install . --group dev - python -m pip install pytest-cov - - name: Run tests - run: |- - ls -lah - cat .coveragerc - pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x - ls -lah - - name: Upload coverage report - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5162c47a..449855f3 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper on: push: + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read @@ -12,7 +18,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +26,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 23fce459..700f3cce 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -1,6 +1,15 @@ name: Test SQLite versions -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -12,10 +21,10 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.13"] sqlite-version: [ #"3", # latest version - "3.46", + #"3.46", #"3.45", #"3.27", #"3.26", @@ -25,7 +34,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -34,7 +43,7 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + uses: ./.github/actions/setup-sqlite-version with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e47db6f..8176a630 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,15 @@ name: Test -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -11,16 +20,20 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + include: + - python-version: "3.14" + coverage: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml + check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) @@ -28,12 +41,27 @@ jobs: run: | pip install . --group dev pip freeze + - name: Install pytest-cov + if: ${{ matrix.coverage }} + run: pip install pytest-cov - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" + if [ "${{ matrix.coverage }}" = "true" ]; then + COV="--cov=datasette --cov-config=.coveragerc" + pytest -n auto -m "not serial" $COV --cov-report= + pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term + else + pytest -n auto -m "not serial" + pytest -m "serial" + fi # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh + - name: Upload coverage report + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml - name: Black run: | black --version diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index a033cd92..f2c074a6 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 72af1eec..5b8818c3 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: diff --git a/.gitignore b/.gitignore index 8c058692..2a7f6620 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ datasets.json scratchpad +ignored/ + .vscode uv.lock diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..58287dd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.11-slim-bookworm AS build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/Justfile b/Justfile index 5fcd9afd..d1b69378 100644 --- a/Justfile +++ b/Justfile @@ -33,10 +33,11 @@ export DATASETTE_SECRET := "not_a_secret" uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt uv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, ruff, cog +# Run linters: black, ruff, prettier, cog @lint: codespell uv run black datasette tests --check uv run ruff check datasette tests + npm run prettier -- --check uv run cog --check README.md docs/*.rst # Apply ruff fixes @@ -48,13 +49,18 @@ export DATASETTE_SECRET := "not_a_secret" uv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: cog blacken-docs +@docs: shots cog blacken-docs uv run make -C docs livehtml # Build docs as static HTML @docs-build: cog blacken-docs rm -rf docs/_build && cd docs && uv run make html +# Take any missing documentation screenshots defined in docs/shots.yml +@shots: + uv run --group shots shot-scraper install + cd docs && uv run --group shots shot-scraper multi shots.yml --no-clobber --reduced-motion --retina + # Apply Black @black: uv run black datasette tests diff --git a/README.md b/README.md index 393e8e5c..1f79778f 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/datasette/__init__.py b/datasette/__init__.py index eb18e59e..982dcc79 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,8 +1,15 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenRestrictions # noqa -from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa +from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor # noqa +from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa +from datasette.utils.asgi import ( # noqa + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 103c616d..587380ed 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -89,7 +89,8 @@ def pytest_runtest_protocol(item, nextitem): continue try: ds.close() - except Exception as e: + except Exception as e: # noqa: BLE001 + # Surfaced as a pytest warning; teardown must not fail the run item.warn( pytest.PytestUnraisableExceptionWarning( f"Error closing Datasette instance: {e!r}" diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 368213af..7503f1d5 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,8 +1,10 @@ -from datasette import hookimpl -from itsdangerous import BadSignature -from datasette.utils import baseconv import time +from itsdangerous import BadSignature + +from datasette import hookimpl +from datasette.utils import baseconv + @hookimpl def actor_from_request(datasette, request): diff --git a/datasette/app.py b/datasette/app.py index 9c9b7de4..3e4c5acf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,8 +1,8 @@ from __future__ import annotations import asyncio -import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from datasette.permissions import Resource @@ -12,11 +12,10 @@ import dataclasses import datetime import functools import glob -import httpx import importlib.metadata import inspect -from itsdangerous import BadSignature import json +import logging import os import re import secrets @@ -28,84 +27,47 @@ import urllib.parse from concurrent import futures from pathlib import Path -from markupsafe import Markup, escape -from itsdangerous import URLSafeSerializer +import httpx2 +from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, - pass_context, PrefixLoader, + pass_context, ) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +from markupsafe import Markup, escape -from .events import Event -from .column_types import SQLiteType from . import stored_queries, write_sql -from .views import Context -from .views.database import ( - database_download, - DatabaseView, - QueryView, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - QueryEditView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) -from .views.index import IndexView -from .views.special import ( - JsonDataView, - PatternPortfolioView, - AutocompleteDebugView, - AuthTokenView, - ApiExplorerView, - CreateTokenView, - LogoutView, - AllowDebugView, - PermissionsDebugView, - MessagesDebugView, - AllowedResourcesView, - PermissionRulesView, - PermissionCheckView, - JumpView, - InstanceSchemaView, - DatabaseSchemaView, - TableSchemaView, -) -from .views.table import ( - TableAutocompleteView, - TableInsertView, - TableUpsertView, - TableSetColumnTypeView, - TableDropView, - TableFragmentView, - table_view, -) -from .views.row import RowView, RowDeleteView, RowUpdateView -from .renderer import json_renderer -from .url_builder import Urls +from .background_tasks import BackgroundTask, BackgroundTaskSupervisor +from .column_types import SQLiteType +from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted - +from .events import Event +from .plugins import DEFAULT_PLUGINS, get_plugins, pm +from .renderer import json_renderer +from .resources import DatabaseResource, TableResource +from .telemetry import ( + TelemetryMiddleware, + _in_datasette_client, + clamp_http_method, + register_datasette, + request_span, + tracer, + unregister_datasette, +) +from .telemetry_registry import HTTP_ROUTE, STARTUP +from .tokens import TokenInvalid +from .tracer import AsgiTracer +from .url_builder import Urls from .utils import ( + SPATIALITE_FUNCTIONS, PaginatedResources, PrefixedUrlString, - SPATIALITE_FUNCTIONS, StartupError, + add_cors_headers, async_call_with_supported_arguments, await_me_maybe, baseconv, @@ -120,48 +82,100 @@ from .utils import ( move_plugins_and_allow, move_table_config, parse_metadata, + redact_keys, resolve_env_secrets, resolve_routes, + row_sql_params_pks, sha256_file, tilde_decode, tilde_encode, to_css_class, urlsafe_components, - redact_keys, - row_sql_params_pks, ) from .utils.asgi import ( AsgiLifespan, + AsgiRunOnFirstRequest, + BadRequest, + DatabaseNotFound, Forbidden, NotFound, - DatabaseNotFound, - TableNotFound, - RowNotFound, Request, Response, - AsgiRunOnFirstRequest, - asgi_static, + RowNotFound, + TableNotFound, asgi_send, asgi_send_file, asgi_send_redirect, + asgi_static, ) -from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, ) -from .tracer import AsgiTracer -from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ - -from .resources import DatabaseResource, TableResource +from .views import Context +from .views.database import ( + DatabaseView, + QueryView, + database_download, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.index import IndexView +from .views.row import RowDeleteView, RowUpdateView, RowView +from .views.special import ( + AllowDebugView, + AllowedResourcesView, + ApiExplorerView, + AuthTokenView, + AutocompleteDebugView, + CreateTokenView, + DatabaseSchemaView, + InstanceSchemaView, + JsonDataView, + JumpView, + LogoutView, + MessagesDebugView, + PatternPortfolioView, + PermissionCheckView, + PermissionRulesView, + PermissionsDebugView, + TableSchemaView, +) +from .views.stored_queries import ( + GlobalQueryListView, + QueryCreateAnalyzeView, + QueryDefinitionView, + QueryDeleteView, + QueryEditView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) +from .views.table import ( + TableAutocompleteView, + TableCountView, + TableDropView, + TableFragmentView, + TableInsertView, + TableSetColumnTypeView, + TableUpsertView, + table_view, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) app_root = Path(__file__).parent.parent +logger = logging.getLogger(__name__) -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + +# _in_datasette_client is defined in telemetry.py to avoid a circular import class _DatasetteClientContext: @@ -181,7 +195,7 @@ class PermissionCheck: """Represents a logged permission check for debugging purposes.""" when: str - actor: Dict[str, Any] | None + actor: dict[str, Any] | None action: str parent: str | None child: str | None @@ -206,6 +220,11 @@ SETTINGS = ( 100, "Maximum rows that can be inserted at a time using the bulk insert API", ), + Setting( + "max_post_body_bytes", + 2 * 1024 * 1024, + "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit", + ), Setting( "num_sql_threads", 3, @@ -306,7 +325,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action, parent, child) + return (actor_key, action.name, parent, action.normalize_child(child)) async def favicon(request, send): @@ -413,6 +432,7 @@ class Datasette: default_deny=False, ): self._startup_invoked = False + self._shutdown_invoked = False self._closed = False assert config_dir is None or isinstance( config_dir, Path @@ -426,7 +446,7 @@ class Datasette: if config_dir: db_files = [] for ext in ("db", "sqlite", "sqlite3"): - db_files.extend(config_dir.glob("*.{}".format(ext))) + db_files.extend(config_dir.glob(f"*.{ext}")) self.files += tuple(str(f) for f in db_files) if ( config_dir @@ -444,8 +464,11 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this + self._setup_db_done = False + self._suppress_background_tasks = False try: self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -453,8 +476,10 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() else: raise + self._background_tasks = BackgroundTaskSupervisor(self) self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: @@ -626,6 +651,8 @@ class Datasette: self.root_enabled = False self.default_deny = default_deny self.client = DatasetteClient(self) + # Last, so metric callbacks never see a partially initialized instance + register_datasette(self) async def apply_metadata_json(self): # Apply any metadata entries from metadata.json to the internal tables @@ -667,10 +694,10 @@ class Datasette: def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: - for environment in pm.hook.jinja2_environment_from_request( + for hook_environment in pm.hook.jinja2_environment_from_request( datasette=self, request=request, env=environment ): - pass + environment = hook_environment return environment def get_action(self, name_or_abbr: str): @@ -724,7 +751,7 @@ class Datasette: catalog_database_names.update( row["database_name"] for row in await internal_db.execute( - "select distinct database_name from {}".format(table) + f"select distinct database_name from {table}" ) if row["database_name"] is not None ) @@ -735,7 +762,7 @@ class Datasette: for stale_db_name in stale_databases: for table in catalog_table_names: conn.execute( - "DELETE FROM {} WHERE database_name = ?".format(table), + f"DELETE FROM {table} WHERE database_name = ?", [stale_db_name], ) @@ -745,19 +772,7 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - placeholders = "(?, ?, ?, ?)" - values = [database_name, str(db.path), db.is_memory, schema_version] - if db.path is None: - placeholders = "(?, null, ?, ?)" - values = [database_name, db.is_memory, schema_version] - await internal_db.execute_write( - """ - INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) - VALUES {} - """.format(placeholders), - values, - ) - await populate_schema_tables(internal_db, db) + await populate_schema_tables(internal_db, db, schema_version) @property def urls(self): @@ -778,61 +793,61 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # Register event classes - event_classes = [] - for hook in pm.hook.register_events(datasette=self): - extra_classes = await await_me_maybe(hook) - if extra_classes: - event_classes.extend(extra_classes) - self.event_classes = tuple(event_classes) + # Group spans created during startup under a single parent span + with tracer.start_as_current_span(STARTUP): + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): - if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): - raise StartupError( - "Duplicate action name: {}".format(action.name) - ) - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): - raise StartupError( - "Duplicate action abbr: {}".format(action.abbr) - ) - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action + # Register actions, but watch out for duplicate name/abbr + action_names = {} + action_abbrs = {} + for hook in pm.hook.register_actions(datasette=self): + if hook: + for action in hook: + if ( + action.name in action_names + and action != action_names[action.name] + ): + raise StartupError(f"Duplicate action name: {action.name}") + if ( + action.abbr + and action.abbr in action_abbrs + and action != action_abbrs[action.abbr] + ): + raise StartupError(f"Duplicate action abbr: {action.abbr}") + action_names[action.name] = action + if action.abbr: + action_abbrs[action.abbr] = action + self.actions[action.name] = action - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError(f"Duplicate column type name: {ct_cls.name}") - self._column_types[ct_cls.name] = ct_cls + # Register column types (classes, not instances) + self._column_types = {} + for hook in pm.hook.register_column_types(datasette=self): + if hook: + for ct_cls in hook: + if ct_cls.name in self._column_types: + raise StartupError( + f"Duplicate column type name: {ct_cls.name}" + ) + self._column_types[ct_cls.name] = ct_cls - for hook in pm.hook.prepare_jinja2_environment( - env=self._jinja_env, datasette=self - ): - await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() - for hook in pm.hook.startup(datasette=self): - await await_me_maybe(hook) - self._startup_invoked = True + for hook in pm.hook.prepare_jinja2_environment( + env=self._jinja_env, datasette=self + ): + await await_me_maybe(hook) + # Ensure internal tables and metadata are populated before startup hooks + await self._refresh_schemas() + await self._save_queries_from_config() + # Load column_types from config into internal DB + await self._apply_column_types_config() + for hook in pm.hook.startup(datasette=self): + await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -865,7 +880,7 @@ class Datasette: actor_id: str, *, expires_after: int | None = None, - restrictions: "TokenRestrictions | None" = None, + restrictions: TokenRestrictions | None = None, handler: str | None = None, ) -> str: """ @@ -905,7 +920,9 @@ class Datasette: Verify an API token by trying all registered token handlers. Returns an actor dict from the first handler that recognizes the - token, or None if no handler accepts it. + token, or None if no handler accepts it. A handler may raise + TokenInvalid for a token it recognizes but rejects (bad signature, + expired) - Datasette turns that into a 401 response. """ for token_handler in self._token_handlers(): result = await token_handler.verify_token(self, token) @@ -920,7 +937,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - name = [key for key in self.databases.keys()][0] + name = next(iter(self.databases.keys())) return self.databases[name] def add_database(self, db, name=None, route=None): @@ -933,7 +950,7 @@ class Datasette: suggestion = name i = 2 while name in self.databases: - name = "{}_{}".format(suggestion, i) + name = f"{suggestion}_{i}" i += 1 db.name = name db.route = route or name @@ -963,18 +980,21 @@ class Datasette: if self._closed: return self._closed = True + # Stop reporting metrics before closing databases + unregister_datasette(self) first_exception = None dbs = list(self.databases.values()) + [self._internal_database] for db in dbs: try: db.close() - except Exception as e: + except Exception as e: # noqa: BLE001 + # Collect the first failure and re-raise after every close() has run if first_exception is None: first_exception = e if self.executor is not None: try: self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: + except Exception as e: # noqa: BLE001 if first_exception is None: first_exception = e if first_exception is not None: @@ -1323,24 +1343,15 @@ class Datasette: actual = ( actual_sqlite_type.value if actual_sqlite_type is not None - else "unrecognized {!r}".format(column_detail.type) + else f"unrecognized {column_detail.type!r}" ) raise ValueError( - "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " - "has SQLite type {}".format( - ct_cls.name, - allowed, - database, - resource, - column, - actual, - ) + f"Column type {ct_cls.name!r} is only applicable to SQLite types {allowed} but {database}.{resource}.{column} " + f"has SQLite type {actual}" ) async def _apply_column_types_config(self): """Load column_types from datasette.json config into the internal DB.""" - import logging - for db_name, db_conf in (self.config or {}).get("databases", {}).items(): for table_name, table_conf in db_conf.get("tables", {}).items(): for col_name, ct in table_conf.get("column_types", {}).items(): @@ -1350,7 +1361,7 @@ class Datasette: col_type = ct["type"] config = ct.get("config") if col_type not in self._column_types: - logging.warning( + logger.warning( "column_types config references unknown type %r " "for %s.%s.%s", col_type, @@ -1363,7 +1374,7 @@ class Datasette: db_name, table_name, col_name, col_type, config ) except ValueError as ex: - logging.warning(str(ex)) + logger.warning(str(ex)) async def get_column_type(self, database: str, resource: str, column: str): """ @@ -1416,7 +1427,7 @@ class Datasette: resource: str, column: str, column_type: str, - config: dict = None, + config: dict | None = None, ) -> None: """Assign a column type. Overwrites any existing assignment.""" ct_cls = self._column_types.get(column_type) @@ -1500,9 +1511,7 @@ class Datasette: possible_names = {plugin["name"], plugin["name"].replace("-", "_")} if plugin_name in possible_names: return _resolve_static_asset_path(plugin["static_path"], path) - raise FileNotFoundError( - "No static assets found for plugin {}".format(plugin_name) - ) + raise FileNotFoundError(f"No static assets found for plugin {plugin_name}") def _static_mounted_asset(self, mount_name, path): mount_name = mount_name.strip("/") @@ -1512,7 +1521,7 @@ class Datasette: _resolve_static_asset_path(dirname, path), self.urls.path("/{}/{}".format(mount_name, path.lstrip("/"))), ) - raise FileNotFoundError("No static mount found for {}".format(mount_name)) + raise FileNotFoundError(f"No static mount found for {mount_name}") def _static_asset_hash(self, filepath): filepath = Path(filepath) @@ -1544,15 +1553,28 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: + # Extension loading is only enabled for as long as it takes to + # load the configured extensions. Leaving it enabled would let + # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) - else: - conn.execute("SELECT load_extension(?)", [extension]) + try: + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + if sys.version_info >= (3, 12): + conn.load_extension(path, entrypoint=entrypoint) + else: + # Connection.load_extension() only gained the + # entrypoint argument in Python 3.12 + conn.execute( + "SELECT load_extension(?, ?)", [path, entrypoint] + ) + else: + conn.load_extension(extension) + finally: + conn.enable_load_extension(False) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1603,18 +1625,17 @@ class Datasette: if await self.allowed(action="view-instance", actor=actor): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link - if database: - if await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), - actor=actor, - ): - crumbs.append( - { - "href": self.urls.database(database), - "label": database, - } - ) + if database and await self.allowed( + action="view-database", + resource=DatabaseResource(database=database), + actor=actor, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) # Table link if table: assert database, "table= requires database=" @@ -1633,7 +1654,7 @@ class Datasette: async def actors_from_ids( self, actor_ids: Iterable[str | int] - ) -> Dict[int | str, Dict]: + ) -> dict[int | str, dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1642,9 +1663,9 @@ class Datasette: return result async def track_event(self, event: Event): - assert isinstance(event, self.event_classes), "Invalid event type: {}".format( - type(event) - ) + assert isinstance( + event, self.event_classes + ), f"Invalid event type: {type(event)}" for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) @@ -1681,7 +1702,7 @@ class Datasette: self, actor: dict, action: str, - resource: "Resource" | None = None, + resource: Resource | None = None, ): """ Check if actor can see a resource and if it's private. @@ -1746,8 +1767,145 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) + if action == "view-table": + sql, params = await self._apply_derived_table_permissions_to_sql( + sql, + params, + actor=actor, + parent=parent, + include_is_private=include_is_private, + ) return ResourcesSQL(sql, params) + async def _allowed_derived_table_source( + self, database, source, *, actor, dependencies + ): + """Check an immediate source, denying sources that are themselves derived.""" + if any( + TableResource.normalize_child(table) + == TableResource.normalize_child(source) + for table in dependencies + ): + return False + # The source has no dependency in this map. Evaluate its own permission + # and prerequisites without starting another dependency check. + verdicts = await self._allowed_many( + actions=["view-table"], + resource=TableResource(database, source), + actor=actor, + check_derived=False, + ) + return verdicts["view-table"] + + async def _apply_derived_table_permissions_to_sql( + self, + sql, + params, + *, + actor, + parent, + include_is_private, + ): + databases = ( + [(parent, self.databases[parent])] + if parent in self.databases + else ([] if parent is not None else list(self.databases.items())) + ) + dependency_maps = dict( + zip( + (name for name, _ in databases), + await asyncio.gather( + *(db.derived_table_dependencies() for _, db in databases) + ), + ) + ) + dependencies = [ + (database_name, child, source) + for database_name, dependency_map in dependency_maps.items() + for child, source in dependency_map.items() + ] + if not dependencies: + return sql, params + + sources = sorted( + {(database_name, source) for database_name, _, source in dependencies} + ) + actor_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=actor, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + actor_allowed = dict(zip(sources, actor_verdicts)) + + anonymous_allowed = {} + if include_is_private: + anonymous_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=None, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + anonymous_allowed = dict(zip(sources, anonymous_verdicts)) + + wrapped_params = dict(params) + derived_rows = [ + [ + database_name, + child, + int(actor_allowed[(database_name, source)]), + *( + [int(anonymous_allowed[(database_name, source)])] + if include_is_private + else [] + ), + ] + for database_name, child, source in dependencies + ] + derived_param = "_datasette_derived_permissions" + while derived_param in wrapped_params: + derived_param += "_" + wrapped_params[derived_param] = json.dumps(derived_rows) + + derived_columns = "parent, child, source_allowed" + select_columns = "allowed.parent, allowed.child, allowed.reason" + if include_is_private: + derived_columns += ", source_anonymous_allowed" + select_columns += ( + ", CASE WHEN derived.source_anonymous_allowed = 0 " + "THEN 1 ELSE allowed.is_private END AS is_private" + ) + wrapped_sql = f""" +WITH derived_permissions({derived_columns}) AS ( + SELECT + json_extract(value, '$[0]'), + json_extract(value, '$[1]'), + json_extract(value, '$[2]') + {", json_extract(value, '$[3]')" if include_is_private else ""} + FROM json_each(:{derived_param}) +), +allowed AS ( +{sql} +) +SELECT {select_columns} +FROM allowed +LEFT JOIN derived_permissions AS derived + ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE +WHERE COALESCE(derived.source_allowed, 1) = 1 +ORDER BY allowed.parent, allowed.child +""".strip() + return wrapped_sql, wrapped_params + async def allowed_resources( self, action: str, @@ -1880,10 +2038,7 @@ class Datasette: if truncated and resources: last_resource = resources[-1] # Use tilde-encoding like table pagination - next_token = "{},{}".format( - tilde_encode(str(last_resource.parent)), - tilde_encode(str(last_resource.child)), - ) + next_token = f"{tilde_encode(str(last_resource.parent))},{tilde_encode(str(last_resource.child))}" return PaginatedResources( resources=resources, @@ -1901,7 +2056,7 @@ class Datasette: self, *, action: str, - resource: "Resource" = None, + resource: Resource = None, actor: dict | None = None, ) -> bool: """ @@ -1932,7 +2087,7 @@ class Datasette: self, *, actions: Sequence[str], - resource: "Resource" = None, + resource: Resource = None, actor: dict | None = None, ) -> dict[str, bool]: """ @@ -1953,11 +2108,17 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ - from datasette.utils.actions_sql import check_permissions_for_actions + return await self._allowed_many( + actions=actions, resource=resource, actor=actor, check_derived=True + ) + + async def _allowed_many(self, *, actions, resource, actor, check_derived): + """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, ) + from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None @@ -1990,7 +2151,7 @@ class Datasette: to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, name, parent, child) + key = _permission_cache_key(actor, self.actions[name], parent, child) if key in cache: final[name] = cache[key] continue @@ -2006,6 +2167,28 @@ class Datasette: child=child, ) + if ( + check_derived + and "view-table" in to_check + and raw.get("view-table") + and isinstance(resource, TableResource) + and parent in self.databases + ): + dependencies = await self.databases[parent].derived_table_dependencies() + source = next( + ( + source + for table, source in dependencies.items() + if TableResource.normalize_child(table) + == TableResource.normalize_child(child) + ), + None, + ) + if source is not None: + raw["view-table"] = await self._allowed_derived_table_source( + parent, source, actor=actor, dependencies=dependencies + ) + def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2023,7 +2206,9 @@ class Datasette: # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[_permission_cache_key(actor, name, parent, child)] = final[name] + cache[ + _permission_cache_key(actor, self.actions[name], parent, child) + ] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2046,7 +2231,7 @@ class Datasette: self, *, action: str, - resource: "Resource" = None, + resource: Resource = None, actor: dict | None = None, ): """ @@ -2100,18 +2285,32 @@ class Datasette: db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) # Find the foreign_key for this column - try: - fk = [ + fk = next( + ( foreign_key for foreign_key in foreign_keys if foreign_key["column"] == column - ][0] - except IndexError: + ), + None, + ) + if fk is None: return {} # Ensure user has permission to view the referenced table from datasette.resources import TableResource other_table = fk["other_table"] + # Foreign key declarations can spell the target with different casing. + target_table = ( + await db.execute( + "select name from sqlite_master where type='table' and name=? collate nocase", + [other_table], + ) + ).first() + if target_table is None: + # SQLite accepts a foreign key to a table that does not exist, and + # linking to it would only lead to a 404 + return {} + other_table = target_table[0] other_column = fk["other_column"] if other_column is None: other_pks = await db.primary_keys(other_table) @@ -2168,6 +2367,18 @@ class Datasette: for name, d in self.databases.items() ] + async def _connected_databases_for_actor(self, actor): + page = await self.allowed_resources("view-database", actor) + allowed_names = {resource.parent async for resource in page.all()} + return [ + database + for database in self._connected_databases() + if database["name"] in allowed_names + ] + + async def _databases_data(self, request): + return {"databases": await self._connected_databases_for_actor(request.actor)} + def _versions(self): conn = sqlite3.connect(":memory:") self._prepare_connection(conn, "_memory") @@ -2182,16 +2393,17 @@ class Datasette: sqlite_extensions[extension] = result.fetchone()[0] else: sqlite_extensions[extension] = None - except Exception: + except Exception: # noqa: BLE001, S110 + # Probing for optional SQLite extensions - absence is the normal case pass # More details on SpatiaLite if "spatialite" in sqlite_extensions: spatialite_details = {} for fn in SPATIALITE_FUNCTIONS: try: - result = conn.execute("select {}()".format(fn)) + result = conn.execute(f"select {fn}()") spatialite_details[fn] = result.fetchone()[0] - except Exception as e: + except sqlite3.Error as e: spatialite_details[fn] = {"error": str(e)} sqlite_extensions["spatialite"] = spatialite_details @@ -2199,9 +2411,7 @@ class Datasette: fts_versions = [] for fts in ("FTS5", "FTS4", "FTS3"): try: - conn.execute( - "CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(fts=fts) - ) + conn.execute(f"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)") fts_versions.append(fts) except sqlite3.OperationalError: continue @@ -2260,7 +2470,7 @@ class Datasette: "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": list(sorted(set(p["hooks"]))), + "hooks": sorted(set(p["hooks"])), } for p in ps ] @@ -2284,6 +2494,21 @@ class Datasette: ) return d + def _tasks(self): + return { + "tasks": [ + { + "name": t.name, + "state": t.state, + "function": t.function, + "started_at": t.started_at, + "exception": repr(t.exception) if t.exception else None, + } + for t in self._background_tasks.tasks() + ], + "launched": self._background_tasks.launched, + } + def _actor(self, request): return {"actor": request.actor} @@ -2336,13 +2561,15 @@ class Datasette: async def render_template( self, - templates: List[str] | str | Template, - context: Dict[str, Any] | Context | None = None, + templates: list[str] | str | Template, + context: dict[str, Any] | Context | None = None, request: Request | None = None, view_name: str | None = None, ): if not self._startup_invoked: - raise Exception("render_template() called before await ds.invoke_startup()") + raise RuntimeError( + "render_template() called before await ds.invoke_startup()" + ) context = context or {} if isinstance(templates, Template): template = templates @@ -2388,9 +2615,11 @@ class Datasette: datasette=self, ): extra_vars = await await_me_maybe(extra_vars) - assert isinstance(extra_vars, dict), "extra_vars is of type {}".format( - type(extra_vars) - ) + if extra_vars is None: + continue + assert isinstance( + extra_vars, dict + ), f"extra_vars is of type {type(extra_vars)}" extra_template_vars.update(extra_vars) async def menu_links(): @@ -2409,29 +2638,27 @@ class Datasette: # the contract tests fail otherwise template_context = { **context, - **{ - "request": request, - "crumb_items": self._crumb_items, - "urls": self.urls, - "actor": request.actor if request else None, - "menu_links": menu_links, - "display_actor": display_actor, - "show_logout": request is not None - and "ds_actor" in request.cookies - and request.actor, - "zip": zip, - "body_scripts": body_scripts, - "format_bytes": format_bytes, - "show_messages": lambda: self._show_messages(request), - "extra_css_urls": await self._asset_urls( - "extra_css_urls", template, context, request, view_name - ), - "extra_js_urls": await self._asset_urls( - "extra_js_urls", template, context, request, view_name - ), - "base_url": self.setting("base_url"), - "datasette_version": __version__, - }, + "request": request, + "crumb_items": self._crumb_items, + "urls": self.urls, + "actor": request.actor if request else None, + "menu_links": menu_links, + "display_actor": display_actor, + "show_logout": request is not None + and "ds_actor" in request.cookies + and request.actor, + "zip": zip, + "body_scripts": body_scripts, + "format_bytes": format_bytes, + "show_messages": lambda: self._show_messages(request), + "extra_css_urls": await self._asset_urls( + "extra_css_urls", template, context, request, view_name + ), + "extra_js_urls": await self._asset_urls( + "extra_js_urls", template, context, request, view_name + ), + "base_url": self.setting("base_url"), + "datasette_version": __version__, **extra_template_vars, } if request and request.args.get("_context") and self.setting("template_debug"): @@ -2452,7 +2679,7 @@ class Datasette: ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + (24 * 60 * 60) + expires_at = int(time.time()) + expire_after data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2514,8 +2741,8 @@ class Datasette: def add_route(view, regex): routes.append((regex, view)) - add_route(IndexView.as_view(self), r"/(\.(?Pjsono?))?$") - add_route(IndexView.as_view(self), r"/-/(\.(?Pjsono?))?$") + add_route(IndexView.as_view(self), r"/(\.(?Pjson))?$") + add_route(IndexView.as_view(self), r"/-/(\.(?Pjson))?$") add_route(permanent_redirect("/-/"), r"/-$") add_route(favicon, "/favicon.ico") @@ -2551,7 +2778,10 @@ class Datasette: ) add_route( JsonDataView.as_view( - self, "plugins.json", self._plugins, needs_request=True + self, + "plugins.json", + self._plugins, + needs_request=True, ), r"/-/plugins(\.(?Pjson))?$", ) @@ -2564,11 +2794,24 @@ class Datasette: r"/-/config(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view(self, "threads.json", self._threads), + JsonDataView.as_view( + self, "threads.json", self._threads, permission="permissions-debug" + ), r"/-/threads(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view(self, "databases.json", self._connected_databases), + JsonDataView.as_view( + self, "tasks.json", self._tasks, permission="permissions-debug" + ), + r"/-/tasks(\.(?Pjson))?$", + ) + add_route( + JsonDataView.as_view( + self, + "databases.json", + self._databases_data, + needs_request=True, + ), r"/-/databases(\.(?Pjson))?$", ) add_route( @@ -2581,7 +2824,7 @@ class Datasette: JsonDataView.as_view( self, "actions.json", - self._actions, + lambda: {"actions": self._actions()}, template="debug_actions.html", permission="permissions-debug", ), @@ -2736,6 +2979,10 @@ class Datasette: TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) + add_route( + TableCountView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", + ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -2789,32 +3036,140 @@ class Datasette: db, table_name, _ = await self.resolve_table(request) pk_values = urlsafe_components(request.url_vars["pks"]) sql, params, pks = await row_sql_params_pks(db, table_name, pk_values) + if len(pk_values) != len(pks): + raise BadRequest( + "URL row identifier does not match the primary key for this table" + ) results = await db.execute(sql, params, truncate=True) row = results.first() if row is None: raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) + async def _startup_sequence(self): + """Idempotently run the full startup sequence: table counts for + immutable databases, then invoke_startup(). Safe to call more than + once and safe to call concurrently - callers block until whichever + call got there first has finished. + + This is the single entry point used by both AsgiLifespan (so + real deployments finish startup before accepting requests) and + AsgiRunOnFirstRequest (the fallback for hosts that never send + lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and + `datasette serve` (cli.py) calls it too. The fast path below checks + both `_startup_invoked` and `_setup_db_done` - not just the former - + so that a bare `await ds.invoke_startup()` made by a caller ahead of + `_startup_sequence()` (which only sets `_startup_invoked`) can't + make this method skip the immutable-database table-count precompute. + """ + if self._startup_invoked and self._setup_db_done: + return + async with self._startup_lock: + if self._startup_invoked and self._setup_db_done: + return + if not self._setup_db_done: + # First time server starts up, calculate table counts for + # immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + self._setup_db_done = True + await self.invoke_startup() + + def add_background_task(self, func, name=None) -> BackgroundTask: + """Register a piece of supervised background work, typically from + a plugin's ``startup`` hook. + + ``func`` must be a coroutine function taking one positional + argument, the ``Datasette`` instance - core calls ``func(self)``. + Callable any time after ``__init__``: if background tasks haven't + launched yet (the common case - most callers are ``startup`` hooks, + which run before launch), this buffers the registration until they + do; if they've already launched (e.g. called from a request + handler after the server is up), the task starts immediately. + + Returns a :class:`~datasette.background_tasks.BackgroundTask` + handle (``.name``, ``.state``, ``.task``, ``.exception``, + ``.started_at``, ``.function``, ``.cancel()``). + + ``name`` defaults to ``func.__qualname__``; on a name collision a + ``-2``, ``-3``, ... suffix is appended, since names are how + ``/-/tasks`` and log messages identify work. + """ + return self._background_tasks.add(func, name=name) + + async def start_background_tasks(self): + """Run startup (if it hasn't run yet) and launch every registered + background task. + + Public entry point for tests, embedders, and headless CLIs (the + ``datasette-rss``-style ``fetch --due`` shape) that want supervised + background tasks without running a server - equivalent to what + happens automatically via ASGI lifespan / the first-request + fallback in a served deployment. + """ + await self.invoke_startup() + await self._background_tasks.launch_all() + + async def _launch_background_tasks(self): + """Idempotently launch every registered background task. Private: + this is the entry point wired into the lifecycle trigger lists + (the second entry in both ``AsgiLifespan`` and + ``AsgiRunOnFirstRequest``'s ``on_startup``, after + ``_startup_sequence``) - not something plugins or embedders should + call directly; use ``add_background_task`` / + ``start_background_tasks`` instead. + + Positioned after ``_startup_sequence`` in both trigger lists so + launch always happens once every plugin's ``startup`` hook has had + a chance to register work - the ordering guarantee that makes + ``add_background_task`` useful. No-ops when + ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its + one-shot TestClient request flows through the full ASGI stack, + including the first-request fallback, but must never launch + long-lived background work). + """ + if self._suppress_background_tasks: + return + await self._background_tasks.launch_all() + + async def invoke_shutdown(self): + """Run the graceful teardown sequence: plugin ``shutdown`` hooks, + then cancel and drain supervised background tasks, then close + every database. + """ + if self._shutdown_invoked: + return + self._shutdown_invoked = True + for hook in pm.hook.shutdown(datasette=self): + try: + await await_me_maybe(hook) + except Exception: + logging.getLogger("datasette").exception("shutdown hook failed") + await self._background_tasks.cancel_all(grace=5.0) + self.close() + def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() - async def setup_db(): - # First time server starts up, calculate table counts for immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - - async def _close_on_shutdown(): - self.close() - asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) + asgi = AsgiLifespan( + asgi, + on_startup=[self._startup_sequence, self._launch_background_tasks], + on_shutdown=[self.invoke_shutdown], + ) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) + asgi = AsgiRunOnFirstRequest( + asgi, + on_startup=[self._startup_sequence, self._launch_background_tasks], + ) + # Outermost, so spans from plugin middleware and first-request + # startup are children of the request span + asgi = TelemetryMiddleware(asgi) return asgi @@ -2847,7 +3202,55 @@ class DatasetteRouter: if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] scope = dict(scope, route_path=path) - request = Request(scope, receive) + request = Request( + scope, + receive, + max_post_body_bytes=self.ds.setting("max_post_body_bytes"), + ) + match, view = resolve_routes(self.routes, path) + is_static = view is favicon or getattr(view, "_datasette_static", False) + original_send = send + + async def send(message): + if message["type"] == "http.response.start" and not ( + is_static and message["status"] in (200, 304) + ): + # Decide privacy after rendering, including for streaming responses + # and error handlers. A public primary resource can still include + # private labels, actor navigation, or cookie-dependent content. + headers = list(message.get("headers", [])) + personalized = ( + request.actor is not None + or "cookie" in request.headers + or "authorization" in request.headers + or any(key.lower() == b"set-cookie" for key, _ in headers) + ) + if personalized: + headers = [ + (key, value) + for key, value in headers + if key.lower() != b"cache-control" + ] + headers.append((b"cache-control", b"private, no-store")) + + # Anonymous responses must not be reused for credentialed requests. + # Preserve any additional variation specified by views or plugins. + vary = [ + part.strip() + for key, value in headers + if key.lower() == b"vary" + for part in value.split(b",") + if part.strip() + ] + if b"*" not in vary: + for name in (b"Cookie", b"Authorization"): + if name.lower() not in {part.lower() for part in vary}: + vary.append(name) + headers = [(k, v) for k, v in headers if k.lower() != b"vary"] + headers.append((b"vary", b", ".join(vary))) + message = dict(message, headers=headers) + await original_send(message) + # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -2867,21 +3270,38 @@ class DatasetteRouter: # Handle authentication default_actor = scope.get("actor") or None actor = None + token_error = None results = pm.hook.actor_from_request(datasette=self.ds, request=request) for result in results: - result = await await_me_maybe(result) + try: + result = await await_me_maybe(result) + except TokenInvalid as ex: + # A presented token was recognized but rejected - fail the + # request with a 401 even if another credential is valid, + # but keep awaiting the remaining coroutines first + if token_error is None: + token_error = ex + continue if result and actor is None: actor = result # Don't break — we must await all coroutines to avoid # "coroutine was never awaited" warnings + if token_error is not None: + return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - - match, view = resolve_routes(self.routes, path) + request.scope = scope if match is None: return await self.handle_404(request, send) + # Now the route is known, add it to the request span + span = request_span(scope) + if span is not None: + route = match.re.pattern + span.set_attribute(HTTP_ROUTE, route) + span.update_name(f"{clamp_http_method(request.method)} {route}") + new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) request.scope = new_scope try: @@ -2902,9 +3322,19 @@ class DatasetteRouter: custom_response ), "Default forbidden() hook should have been called" return await custom_response.asgi_send(send) - except Exception as exception: + except Exception as exception: # noqa: BLE001 + # This IS the top-level error handler - it must catch everything return await self.handle_exception(request, send, exception) + async def handle_401(self, request, send, exception): + # A presented bearer token was recognized by a handler but rejected. + # Bearer tokens are API credentials, so this is always JSON. + headers = {"www-authenticate": 'Bearer error="invalid_token"'} + if self.ds.cors: + add_cors_headers(headers) + response = Response.error([str(exception)], 401, headers=headers) + await response.asgi_send(send) + async def handle_404(self, request, send, exception=None): # If path contains % encoding, redirect to tilde encoding if "%" in request.path: @@ -2915,7 +3345,7 @@ class DatasetteRouter: request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E") ) if request.query_string: - new_path += "?{}".format(request.query_string) + new_path += f"?{request.query_string}" await asgi_send_redirect(send, new_path) return # If URL has a trailing slash, redirect to URL without it @@ -3125,8 +3555,7 @@ _curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): # Drop the ".html" suffix - if filepath.endswith(".html"): - filepath = filepath[: -len(".html")] + filepath = filepath.removesuffix(".html") re_bits = ["/"] for bit in _curly_re.split(filepath): if _curly_re.match(bit): @@ -3183,14 +3612,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3237,10 +3666,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx + **kwargs: Additional arguments to pass to httpx2 Returns: - httpx.Response: The response from the request + httpx2.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3249,16 +3678,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/background_tasks.py b/datasette/background_tasks.py new file mode 100644 index 00000000..6b34fd85 --- /dev/null +++ b/datasette/background_tasks.py @@ -0,0 +1,227 @@ +""" +Supervised background-task registration for Datasette core. + +Plugins that need long-lived background work (a polling loop, a queue +consumer, a scheduled job runner) register it with +``datasette.add_background_task(func, name=None)`` - typically from a +``startup`` plugin hook - instead of fire-and-forgetting their own +``asyncio.create_task()``. Core owns: + +- **references**: every launched ``asyncio.Task`` is kept alive on a + :class:`BackgroundTaskSupervisor`, so it can never be silently garbage + collected the way an unreferenced ``create_task()`` call can be; +- **launch timing**: registered work is buffered until + :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to + happen only after *every* plugin's ``startup`` hook has finished - so + a task that depends on another plugin having registered something first + doesn't need ``tryfirst=True`` ordering tricks; +- **crash surfacing**: an unhandled exception in a background task is + logged with its full traceback to the ``datasette.background_tasks`` + logger and recorded on the handle, instead of becoming an "Task + exception was never retrieved" warning nobody sees; +- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels + every task still running and waits (with a grace period) for them to + actually stop. +""" + +from __future__ import annotations + +import asyncio +import datetime +import functools +import logging +from collections.abc import Awaitable, Callable + +logger = logging.getLogger("datasette.background_tasks") + + +def _utcnow_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _function_path(func: Callable) -> str: + """Describe the callable without guessing which plugin registered it.""" + while isinstance(func, functools.partial): + func = func.func + if not hasattr(func, "__qualname__"): + func = type(func).__call__ + return f"{func.__module__}.{func.__qualname__}" + + +class BackgroundTask: + """A handle to a single piece of supervised background work. + + States: ``registered`` (added but not yet launched) -> ``running`` -> + one of ``completed`` (returned cleanly), ``crashed`` (raised an + exception other than ``CancelledError`` - see ``.exception``), or + ``cancelled`` (``.cancel()`` was called, or it was still running at + shutdown). + """ + + def __init__( + self, + name: str, + func: Callable[[object], Awaitable[None]], + ): + self.name = name + self.state = "registered" + self.task: asyncio.Task | None = None + self.exception: BaseException | None = None + self.started_at: str | None = None + self.function = _function_path(func) + self._func = func + self._supervisor: BackgroundTaskSupervisor | None = None + + def cancel(self) -> None: + """Cancel this task. + + If it has already been launched, cancels the underlying + ``asyncio.Task`` - its state becomes ``cancelled`` once the + cancellation is observed (asynchronously, via the task's done + callback). If it has not been launched yet, this is a no-op as + far as asyncio is concerned (there's no task to cancel) but it + deregisters the handle from its supervisor so it never runs. + """ + if self.task is not None: + self.task.cancel() + elif self._supervisor is not None: + self._supervisor._deregister(self) + + def __repr__(self) -> str: + return f"" + + +class BackgroundTaskSupervisor: + """Owns registration and launch of every :class:`BackgroundTask` for a + single ``Datasette`` instance. + + Registration (:meth:`add`) is separate from launch + (:meth:`launch_all`): plugins register work whenever convenient + (typically from a ``startup`` hook, but request handlers can register + dynamic per-job work too), and it either sits buffered until + :meth:`launch_all` runs, or - if :meth:`launch_all` has already run - + starts immediately. + + Strong references to every :class:`BackgroundTask` (and its + ``asyncio.Task``) are kept for the life of the instance, by design - + that's what makes the enrichments-style "fire-and-forget task gets + garbage collected mid-flight" bug impossible here. There is currently + no pruning of completed/crashed/cancelled tasks, so a plugin that + dynamically registers many short-lived tasks over a long process + lifetime (a per-job registration pattern, e.g. one task per queued + job) will grow this list without bound. That's an accepted v1 + trade-off in favour of full introspection (``/-/tasks``); revisit + with a pruning or capping policy if unbounded growth is reported in + practice. + """ + + def __init__(self, datasette): + self._datasette = datasette + self._tasks: list[BackgroundTask] = [] + self._names = set() + self._launched = False + self._lock = asyncio.Lock() + + def add(self, func, name=None) -> BackgroundTask: + base_name = name or getattr(func, "__qualname__", None) or repr(func) + actual_name = self._unique_name(base_name) + handle = BackgroundTask(actual_name, func) + handle._supervisor = self + self._tasks.append(handle) + self._names.add(actual_name) + if self._launched: + self._launch_one(handle) + return handle + + def _unique_name(self, base_name: str) -> str: + if base_name not in self._names: + return base_name + n = 2 + while f"{base_name}-{n}" in self._names: + n += 1 + return f"{base_name}-{n}" + + def _deregister(self, handle: BackgroundTask) -> None: + try: + self._tasks.remove(handle) + except ValueError: + pass + self._names.discard(handle.name) + + def _launch_one(self, handle: BackgroundTask) -> None: + handle.state = "running" + handle.started_at = _utcnow_iso() + handle.task = asyncio.create_task( + handle._func(self._datasette), name=handle.name + ) + handle.task.add_done_callback(functools.partial(_on_task_done, handle)) + + async def launch_all(self) -> None: + """Launch every currently-registered task that hasn't launched + yet. Idempotent and safe to call concurrently: subsequent (or + racing) calls are no-ops once the first has set ``self._launched``. + """ + if self._launched: + return + async with self._lock: + if self._launched: + return + self._launched = True + for handle in list(self._tasks): + if handle.task is None: + self._launch_one(handle) + + async def cancel_all(self, grace: float = 5.0) -> None: + """Cancel every task that isn't already done, then wait up to + ``grace`` seconds for them to actually finish. Stragglers still + running after that are logged by name (but left to finish or not + on their own - this does not forcibly kill them, asyncio has no + mechanism for that). + """ + handles_by_task = { + handle.task: handle for handle in self._tasks if handle.task is not None + } + pending = [task for task in handles_by_task if not task.done()] + for task in pending: + task.cancel() + if not pending: + return + _done, not_done = await asyncio.wait(pending, timeout=grace) + if not_done: + names = sorted(handles_by_task[task].name for task in not_done) + logger.warning( + "%d background task(s) did not finish within the %.1fs grace " + "period after cancellation: %s", + len(names), + grace, + ", ".join(names), + ) + + def tasks(self) -> list[BackgroundTask]: + """Return every registered :class:`BackgroundTask`, launched or + not, in registration order. Used by the ``/-/tasks`` debug + endpoint. + """ + return list(self._tasks) + + @property + def launched(self) -> bool: + """Whether :meth:`launch_all` has run yet - lets ``/-/tasks`` + distinguish "no tasks registered" from "tasks registered but + nothing has armed the launch yet" without reaching for the + private ``_launched`` attribute. + """ + return self._launched + + +def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None: + if task.cancelled(): + handle.state = "cancelled" + return + exc = task.exception() + if exc is not None: + handle.state = "crashed" + handle.exception = exc + logger.error("Background task %r crashed", handle.name, exc_info=exc) + return + handle.state = "completed" diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index 4d8c6bea..b6c8b77f 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -1,8 +1,9 @@ -from datasette import hookimpl -from datasette.utils.asgi import Response, BadRequest -from datasette.utils import to_css_class import hashlib +from datasette import hookimpl +from datasette.utils import to_css_class +from datasette.utils.asgi import BadRequest, Response + _BLOB_COLUMN = "_blob_column" _BLOB_HASH = "_blob_hash" diff --git a/datasette/cli.py b/datasette/cli.py index 90a33e80..e83de93a 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -1,43 +1,45 @@ import asyncio -import uvicorn -import click -from click import formatting -from click.types import CompositeParamType -from click_default_group import DefaultGroup import functools import json import os import pathlib -from runpy import run_module import shutil -from subprocess import call import sys import textwrap import webbrowser +from runpy import run_module +from subprocess import call + +import click +import uvicorn +from click import formatting +from click.types import CompositeParamType +from click_default_group import DefaultGroup + from .app import ( - Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, + Datasette, pm, ) from .inspect import inspect_tables from .utils import ( + ConnectionProblem, LoadExtension, + SpatialiteConnectionProblem, + SpatialiteNotFound, StartupError, + StaticMount, + ValueAsBooleanError, check_connection, deep_dict_update, find_spatialite, - parse_metadata, - ConnectionProblem, - SpatialiteConnectionProblem, initial_path_for_datasette, pairs_to_nested_config, + parse_metadata, temporary_docker_directory, value_as_boolean, - SpatialiteNotFound, - StaticMount, - ValueAsBooleanError, ) from .utils.sqlite import sqlite3 from .utils.testing import TestClient @@ -75,7 +77,7 @@ class Setting(CompositeParamType): # Datasette 1.0, we turn bare setting names into setting.name # Type checking for those older settings default = DEFAULT_SETTINGS[name] - name = "settings.{}".format(name) + name = f"settings.{name}" if isinstance(default, bool): try: return name, "true" if value_as_boolean(value) else "false" @@ -155,7 +157,11 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) + + def _inspect_tables(conn): + return inspect_tables(conn, {}) + + tables = await database.execute_fn(_inspect_tables) data[name] = { "hash": database.hash, "size": database.size, @@ -171,7 +177,6 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" - pass # Register publish plugins @@ -496,6 +501,7 @@ def uninstall(packages, yes): "--internal", type=click.Path(), help="Path to a persistent Datasette internal SQLite database", + envvar="DATASETTE_INTERNAL", ) def serve( files, @@ -578,27 +584,27 @@ def serve( # https://github.com/simonw/datasette/issues/2389 deep_dict_update(config_data, settings_updates) - kwargs = dict( - immutables=immutable, - cache_headers=not reload, - cors=cors, - inspect_data=inspect_data, - config=config_data, - metadata=metadata_data, - sqlite_extensions=sqlite_extensions, - template_dir=template_dir, - plugins_dir=plugins_dir, - static_mounts=static, - settings=None, # These are passed in config= now - memory=memory, - secret=secret, - version_note=version_note, - pdb=pdb, - crossdb=crossdb, - nolock=nolock, - internal=internal, - default_deny=default_deny, - ) + kwargs = { + "immutables": immutable, + "cache_headers": not reload, + "cors": cors, + "inspect_data": inspect_data, + "config": config_data, + "metadata": metadata_data, + "sqlite_extensions": sqlite_extensions, + "template_dir": template_dir, + "plugins_dir": plugins_dir, + "static_mounts": static, + "settings": None, # These are passed in config= now + "memory": memory, + "secret": secret, + "version_note": version_note, + "pdb": pdb, + "crossdb": crossdb, + "nolock": nolock, + "internal": internal, + "default_deny": default_deny, + } # Separate directories from files directories = [f for f in files if os.path.isdir(f)] @@ -621,9 +627,7 @@ def serve( conn.close() else: raise click.ClickException( - "Invalid value for '[FILES]...': Path '{}' does not exist.".format( - file - ) + f"Invalid value for '[FILES]...': Path '{file}' does not exist." ) # Check for duplicate files by resolving all paths to their absolute forms @@ -664,16 +668,6 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run async soundness checks before startup hooks, since invoke_startup - # now populates internal tables which requires querying each database - run_sync(lambda: check_databases(ds)) - - # Run the "startup" plugin hooks - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -681,10 +675,23 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: + # --get means we don't run Uvicorn at all + run_sync(lambda: check_databases(ds)) + + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + + # --get never launches background tasks: TestClient's request below + # flows through the full ASGI stack, including the + # AsgiRunOnFirstRequest fallback, which would otherwise launch them. + ds._suppress_background_tasks = True + client = TestClient(ds) request_headers = {} if token: - request_headers["Authorization"] = "Bearer {}".format(token) + request_headers["Authorization"] = f"Bearer {token}" cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) @@ -705,30 +712,54 @@ def serve( sys.exit(exit_code) return - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = run_sync(lambda: initial_path_for_datasette(ds)) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = dict( - host=host, port=port, log_level="info", lifespan="on", workers=1 - ) - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - uvicorn.run(ds.app(), **uvicorn_kwargs) + # check_databases, invoke_startup() and the uvicorn server all run on a + # single event loop, so that anything a plugin's "startup" hook schedules + # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is + # still alive when the server starts handling requests. + async def _serve_async(): + # Populate internal catalog tables before invoke_startup + await check_databases(ds) + + # Run the full startup sequence (immutable-database table-count + # precompute + the "startup" plugin hooks) via the same entry point + # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when + # uvicorn's lifespan.startup fires moments later. + try: + await ds._startup_sequence() + except StartupError as e: + raise click.ClickException(e.args[0]) + + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = await initial_path_for_datasette(ds) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) + await server.serve() + + asyncio.run(_serve_async()) @cli.command() @@ -885,7 +916,7 @@ async def check_databases(ds): ) except ConnectionProblem as e: raise click.UsageError( - f"Connection to {database.path} failed check: {str(e.args[0])}" + f"Connection to {database.path} failed check: {e.args[0]!s}" ) # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning if ( @@ -893,9 +924,5 @@ async def check_databases(ds): and len([db for db in ds.databases.values() if not db.is_memory]) > SQLITE_LIMIT_ATTACHED ): - msg = ( - "Warning: --crossdb only works with the first {} attached databases".format( - SQLITE_LIMIT_ATTACHED - ) - ) + msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases" click.echo(click.style(msg, bold=True, fg="yellow"), err=True) diff --git a/datasette/column_types.py b/datasette/column_types.py index 11a14ec0..92fdd969 100644 --- a/datasette/column_types.py +++ b/datasette/column_types.py @@ -64,14 +64,14 @@ class ColumnType: Return an HTML string to render this cell value, or None to fall through to the default render_cell plugin hook chain. """ - return None + return async def validate(self, value, datasette): """ Validate a value before it is written. Return None if valid, or a string error message if invalid. """ - return None + return async def transform_value(self, value, datasette): """ diff --git a/datasette/csrf.py b/datasette/csrf.py index df239aee..a62f9473 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -40,12 +40,12 @@ def _origin_tuple(value): scheme = (parsed.scheme or "").lower() host = (parsed.hostname or "").lower() if not scheme or not host: - raise ValueError("missing scheme or host in {!r}".format(value)) + raise ValueError(f"missing scheme or host in {value!r}") port = parsed.port # may raise ValueError on bad ports if port is None: port = DEFAULT_PORTS.get(scheme) if port is None: - raise ValueError("unknown default port for scheme {!r}".format(scheme)) + raise ValueError(f"unknown default port for scheme {scheme!r}") return scheme, host, port @@ -125,9 +125,7 @@ class CrossOriginProtectionMiddleware: return await self._forbid( send, - "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( - sec_fetch_site - ), + f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'", ) return @@ -141,11 +139,11 @@ class CrossOriginProtectionMiddleware: request_scheme = self._request_scheme(scope) try: origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) + expected_tuple = _origin_tuple(f"{request_scheme}://{host}") except ValueError: await self._forbid( send, - "Malformed Origin {!r} or Host {!r}".format(origin, host), + f"Malformed Origin {origin!r} or Host {host!r}", ) return @@ -155,7 +153,7 @@ class CrossOriginProtectionMiddleware: await self._forbid( send, - "Origin {!r} does not match Host {!r}".format(origin, host), + f"Origin {origin!r} does not match Host {host!r}", ) def _request_scheme(self, scope): @@ -163,7 +161,8 @@ class CrossOriginProtectionMiddleware: try: if self.datasette.setting("force_https_urls"): return "https" - except Exception: + except Exception: # noqa: BLE001, S110 + # Settings may not be readable this early; fall back to the ASGI scheme pass return scope.get("scheme") or "http" diff --git a/datasette/database.py b/datasette/database.py index e7fe1ed9..542b3012 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,33 +1,71 @@ import asyncio import atexit -from collections import namedtuple +import contextvars import inspect import os -from pathlib import Path import queue -import sqlite_utils import sys import tempfile import threading +import time import uuid +from collections import namedtuple +from pathlib import Path +import sqlite_utils +from opentelemetry import context as otel_context_api +from opentelemetry.trace import Status, StatusCode + +from .inspect import inspect_hash +from .telemetry import ( + callback_name, + linked_root_span_kwargs, + record_operation_duration, + record_query_interrupted, + record_write_queue_wait, + sql_attribute, + sql_operation_name, + tracer, +) +from .telemetry_registry import ( + CALLBACK, + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY, + DB_QUERY_EXECUTE, + DB_QUERY_TEXT, + DB_SYSTEM, + DB_WRITE_EXECUTE, + DB_WRITE_QUEUE_WAIT, + EXECUTEMANY, + EXECUTESCRIPT, + INTERRUPTED, + ISOLATED_CONNECTION, + PARAM_COUNT, + PARAM_SETS, + ROWS_RETURNED, + SQL_ERROR_SUPPRESSED, + TIME_LIMIT_MS, + TRANSACTION, + TRUNCATED, +) from .tracer import trace from .utils import ( call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, + escape_sqlite, get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, - sqlite_timelimit, sqlite3, - table_columns, + sqlite_timelimit, table_column_details, + table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_hidden_table_names -from .inspect import inspect_hash +from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names connections = threading.local() @@ -83,6 +121,7 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None + self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -91,16 +130,15 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - # This is used to track all file connections so they can be closed - self._all_file_connections = [] + # Track file and memory connections, including reads on worker threads, + # so close() can release all of them from the calling thread. + self._all_connections = [] if not is_temp_disk: self.mode = mode def _check_not_closed(self): if self._closed: - raise DatasetteClosedError( - "Database {!r} has been closed".format(self.name) - ) + raise DatasetteClosedError(f"Database {self.name!r} has been closed") def _remove_pending_execute_future(self, future): with self._pending_execute_futures_lock: @@ -139,15 +177,18 @@ class Database: if write: extra_kwargs["isolation_level"] = "IMMEDIATE" if self.memory_name: - uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) + uri = f"file:{self.memory_name}?mode=memory&cache=shared" conn = sqlite3.connect( uri, uri=True, check_same_thread=False, **extra_kwargs ) if not write: conn.execute("PRAGMA query_only=1") + self._all_connections.append(conn) return conn if self.is_memory: - return sqlite3.connect(":memory:", uri=True) + conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False) + self._all_connections.append(conn) + return conn # mode=ro or immutable=1? if self.is_mutable: @@ -164,7 +205,7 @@ class Database: conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) - self._all_file_connections.append(conn) + self._all_connections.append(conn) if self.is_temp_disk and not self._wal_enabled: conn.execute("PRAGMA journal_mode=WAL") self._wal_enabled = True @@ -192,23 +233,22 @@ class Database: write_thread.join(timeout=10) if write_thread.is_alive(): sys.stderr.write( - "Datasette: write thread for {!r} did not exit within 10s\n".format( - self.name - ) + f"Datasette: write thread for {self.name!r} did not exit within 10s\n" ) sys.stderr.flush() for future in pending_execute_futures: try: future.result() - except Exception: + except Exception: # noqa: BLE001, S110 + # Shutdown teardown - a failed pending write must not block close() pass - # Close anything still tracked in _all_file_connections - for connection in self._all_file_connections: + # Close anything still tracked in _all_connections + for connection in self._all_connections: try: connection.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass - self._all_file_connections = [] + self._all_connections = [] # Drop per-thread cached read connections we can reach try: delattr(connections, self._thread_local_id) @@ -218,13 +258,13 @@ class Database: if self._read_connection is not None: try: self._read_connection.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass self._read_connection = None if self._write_connection is not None: try: self._write_connection.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass self._write_connection = None if self.is_temp_disk: @@ -246,19 +286,46 @@ class Database: request=None, return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, + transaction=True, + time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def _inner(conn): + def execute_sql(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn(_inner, block=block, request=request) + def _inner(conn): + try: + if time_limit_ms is None: + return execute_sql(conn) + with sqlite_timelimit(conn, time_limit_ms): + return execute_sql(conn) + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + raise + + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), params=params + ): + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute(DB_OPERATION_NAME, operation_name) + if params: + span.set_attribute(PARAM_COUNT, len(params)) + with record_operation_duration(self.name, "write"): + results = await self._execute_write_fn( + _inner, block=block, request=request, transaction=transaction + ) return results async def execute_write_script(self, sql, block=True, request=None): @@ -267,10 +334,19 @@ class Database: def _inner(conn): return conn.executescript(sql) - with trace("sql", database=self.name, sql=sql.strip(), executescript=True): - results = await self.execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), executescript=True + ): + # No db.operation.name, since the script can contain multiple statements + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(EXECUTESCRIPT, True) + with record_operation_duration(self.name, "write"): + results = await self._execute_write_fn( + _inner, block=block, transaction=False, request=request + ) return results async def execute_write_many(self, sql, params_seq, block=True, request=None): @@ -290,9 +366,19 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - results, count = await self.execute_write_fn( - _inner, block=block, request=request - ) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(EXECUTEMANY, True) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute(DB_OPERATION_NAME, operation_name) + with record_operation_duration(self.name, "write"): + results, count = await self._execute_write_fn( + _inner, block=block, request=request + ) + span.set_attribute(PARAM_SETS, count) kwargs["count"] = count return results @@ -309,31 +395,58 @@ class Database: finally: isolated_connection.close() try: - self._all_file_connections.remove(isolated_connection) + self._all_connections.remove(isolated_connection) except ValueError: - # Was probably a memory connection + # May already have been cleared by close(). pass - if self.ds.executor is None: - # non-threaded mode - return _run() - if not write: - # Immutable database - no writes can ever occur, so there is no - # write queue to block; run against a fresh read-only connection - return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, _run - ) - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, callback_name(fn)) + # Immutable databases run this on the read pool, not the write queue + with record_operation_duration(self.name, "write" if write else "read"): + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there + # is no write queue to block; run against a fresh + # read-only connection + ctx = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, ctx.run, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() - return await self.execute_isolated_fn( - lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) - ) + def _analyze_sql(conn): + return analyze_sql_tables(conn, sql, params, database_name=self.name) + + return await self.execute_isolated_fn(_analyze_sql) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): + """Run `fn(conn)` on the write connection, traced as a `db.query` span. + + The SQL-string write methods call `_execute_write_fn()` directly to + avoid creating a second span. + """ + self._check_not_closed() + # Record the name before _wrap_fn_with_hooks() wraps fn + name = callback_name(fn) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, name) + with record_operation_duration(self.name, "write"): + return await self._execute_write_fn( + fn, block=block, transaction=transaction, request=request + ) + + async def _execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() pending_events = [] @@ -348,9 +461,19 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: + self._write_connection.execute("BEGIN IMMEDIATE") result = fn(self._write_connection) else: result = fn(self._write_connection) + if not block: + # There is no write thread here, so the write has already + # finished. Hand back the same (task_id, reply_future) shape + # _send_to_write_thread() returns, with the future already + # resolved, so the block=False path below is identical in + # both modes. + reply_future = asyncio.get_running_loop().create_future() + reply_future.set_result(result) + result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -366,7 +489,8 @@ class Database: async def _dispatch_events_after_write(): try: await reply_future - except Exception: + except Exception: # noqa: BLE001 + # The write failed; skip success events regardless of why # if the write failed, don't emit success events return for event in pending_events: @@ -419,15 +543,24 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) - self._write_thread.name = "_execute_writes for database {}".format( - self.name - ) + self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + task_id = uuid.uuid4() loop = asyncio.get_running_loop() reply_future = loop.create_future() + # Capture the OpenTelemetry context and enqueue time for the write thread self._write_queue.put( - WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) + WriteTask( + fn, + task_id, + loop, + reply_future, + isolated_connection, + transaction, + otel_context_api.get_current(), + time.time_ns(), + block, + ) ) if block: return await reply_future @@ -441,8 +574,11 @@ class Database: conn = None try: conn = self.connect(write=True) + # Threads do not inherit the caller's context, so any spans + # created by prepare_connection hooks here are root spans self.ds._prepare_connection(conn, self.name) - except Exception as e: + except Exception as e: # noqa: BLE001 + # Stored and re-raised to whoever queues the next write conn_exception = e while True: task = self._write_queue.get() @@ -450,43 +586,105 @@ class Database: if conn is not None: try: conn.close() - except Exception: + except Exception: # noqa: BLE001, S110 + # Best-effort close as the write thread exits pass return - exception = None - result = None - if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - try: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e + # block=True: the caller awaits the result, so the write spans + # are children of the caller's span. The token must be detached + # in the finally block or the context leaks into later writes. + # block=False: the caller may finish first, so the write spans + # are root spans with a link back to the caller's span. + token = None + write_span_kwargs = {} + if task.block: + token = otel_context_api.attach(task.otel_context) else: - try: - if task.transaction: - with conn: - result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) + write_span_kwargs = linked_root_span_kwargs(task.otel_context) + try: + exception = None + result = None + # Span covers the time from enqueue to dequeue + dequeued_at_ns = time.time_ns() + tracer.start_span( + DB_WRITE_QUEUE_WAIT, + start_time=task.enqueued_at_ns, + **write_span_kwargs, + ).end(end_time=dequeued_at_ns) + record_write_queue_wait(self.name, dequeued_at_ns - task.enqueued_at_ns) + if conn_exception is not None: + exception = conn_exception + elif task.isolated_connection: + try: + with tracer.start_as_current_span( + DB_WRITE_EXECUTE, **write_span_kwargs + ) as span: + span.set_attribute( + ISOLATED_CONNECTION, + task.isolated_connection, + ) + span.set_attribute(TRANSACTION, task.transaction) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_connections.remove(isolated_connection) + except ValueError: + # May already have been cleared by close(). + pass + except Exception as e: # noqa: BLE001 + # Write thread must survive any task failure or the database wedges + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + else: + try: + with tracer.start_as_current_span( + DB_WRITE_EXECUTE, **write_span_kwargs + ) as span: + span.set_attribute( + ISOLATED_CONNECTION, + task.isolated_connection, + ) + span.set_attribute(TRANSACTION, task.transaction) + if task.transaction: + with conn: + conn.execute("BEGIN IMMEDIATE") + result = task.fn(conn) + else: + result = task.fn(conn) + except Exception as e: # noqa: BLE001 + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) + finally: + if token is not None: + otel_context_api.detach(token) async def execute_fn(self, fn): + """Run `fn(conn)` on a read connection, traced as a `db.query` span. + + `execute()` calls `_execute_fn()` directly to avoid creating a second + span. + """ + self._check_not_closed() + + def fn_in_execute_span(conn): + # Runs on the worker thread + with tracer.start_as_current_span(DB_QUERY_EXECUTE): + return fn(conn) + + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, callback_name(fn)) + with record_operation_duration(self.name, "read"): + return await self._execute_fn(fn_in_execute_span) + + async def _execute_fn(self, fn): self._check_not_closed() if self.ds.executor is None: # non-threaded mode @@ -506,7 +704,11 @@ class Database: with self._pending_execute_futures_lock: self._check_not_closed() - future = self.ds.executor.submit(in_thread) + # Run in a copy of the caller's context so spans created in the + # thread have the correct parent. This needs a fresh copy for + # each submit, since a Context cannot be entered concurrently. + ctx = contextvars.copy_context() + future = self.ds.executor.submit(ctx.run, in_thread) self._pending_execute_futures.add(future) future.add_done_callback(self._remove_pending_execute_future) return await asyncio.wrap_future(future) @@ -523,46 +725,101 @@ class Database: """Executes sql against db_name in a thread""" self._check_not_closed() page_size = page_size or self.ds.page_size + time_limit_ms = self.ds.sql_time_limit_ms + # Callers that pass a shorter custom_time_limit, such as table counts + # and facet suggestions, expect timeouts, so they are not span errors + timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms + if timeout_expected: + time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): - time_limit_ms = self.ds.sql_time_limit_ms - if custom_time_limit and custom_time_limit < time_limit_ms: - time_limit_ms = custom_time_limit - - with sqlite_timelimit(conn, time_limit_ms): + # Expected timeouts and errors with log_sql_errors=False are not + # recorded as span errors, so exceptions are handled explicitly + with tracer.start_as_current_span( + DB_QUERY_EXECUTE, + record_exception=False, + set_status_on_exception=False, + ) as execute_span: try: - cursor = conn.cursor() - cursor.execute(sql, params if params is not None else {}) - max_returned_rows = self.ds.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params if params is not None else {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + sys.stderr.write( + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + ) + sys.stderr.flush() + raise + except QueryInterrupted as e: + if not timeout_expected: + execute_span.record_exception(e) + execute_span.set_status(Status(StatusCode.ERROR, str(e))) + raise + except Exception as e: if log_sql_errors: - sys.stderr.write( - "ERROR: conn={}, sql = {}, params = {}: {}\n".format( - conn, repr(sql), params, e - ) - ) - sys.stderr.flush() + execute_span.record_exception(e) + execute_span.set_status(Status(StatusCode.ERROR, str(e))) raise - if truncate: - return Results(rows, truncated, cursor.description) + if truncate: + return Results(rows, truncated, cursor.description) - else: - return Results(rows, False, cursor.description) + else: + return Results(rows, False, cursor.description) - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_fn(sql_operation_in_thread) + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), params=params + ): + with tracer.start_as_current_span( + DB_QUERY, + kind=DB_QUERY.kind, + record_exception=False, + set_status_on_exception=False, + ) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(TIME_LIMIT_MS, time_limit_ms) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute(DB_OPERATION_NAME, operation_name) + if params: + span.set_attribute(PARAM_COUNT, len(params)) + try: + with record_operation_duration(self.name, "read"): + results = await self._execute_fn(sql_operation_in_thread) + except QueryInterrupted as e: + span.set_attribute(INTERRUPTED, True) + if not timeout_expected: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) + record_query_interrupted(self.name) + raise + except Exception as e: + # log_sql_errors=False callers, such as facet suggestion, + # expect some queries to fail + if log_sql_errors: + span.record_exception(e) + span.set_status(Status(StatusCode.ERROR, str(e))) + else: + span.set_attribute(SQL_ERROR_SUPPRESSED, True) + raise + span.set_attribute(TRUNCATED, results.truncated) + span.set_attribute(ROWS_RETURNED, len(results.rows)) return results @property @@ -603,7 +860,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", + f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})", custom_time_limit=limit, ) ).rows[0][0] @@ -653,17 +910,32 @@ class Database: ) return [r[0] for r in results.rows] + # Named functions rather than lambdas give more useful datasette.callback + # span attributes + async def table_columns(self, table): - return await self.execute_fn(lambda conn: table_columns(conn, table)) + def _table_columns(conn): + return table_columns(conn, table) + + return await self.execute_fn(_table_columns) async def table_column_details(self, table): - return await self.execute_fn(lambda conn: table_column_details(conn, table)) + def _table_column_details(conn): + return table_column_details(conn, table) + + return await self.execute_fn(_table_column_details) async def primary_keys(self, table): - return await self.execute_fn(lambda conn: detect_primary_keys(conn, table)) + def _primary_keys(conn): + return detect_primary_keys(conn, table) + + return await self.execute_fn(_primary_keys) async def fts_table(self, table): - return await self.execute_fn(lambda conn: detect_fts(conn, table)) + def _fts_table(conn): + return detect_fts(conn, table) + + return await self.execute_fn(_fts_table) async def label_column_for_table(self, table): explicit_label_column = (await self.ds.table_config(self.name, table)).get( @@ -707,9 +979,9 @@ class Database: column_names and len(column_names) == 2 and ("id" in column_names or "pk" in column_names) - and not set(column_names) == {"id", "pk"} + and set(column_names) != {"id", "pk"} ): - return [c for c in column_names if c not in ("id", "pk")][0] + return next(c for c in column_names if c not in ("id", "pk")) # Couldn't find a label: return None @@ -755,6 +1027,17 @@ class Database: return hidden_tables + async def derived_table_dependencies(self): + """Return implementation tables and the tables they derive from.""" + schema_version = (await self.execute("PRAGMA schema_version")).first()[0] + if ( + self._cached_derived_table_dependencies is None + or self._cached_derived_table_dependencies[0] != schema_version + ): + dependencies = await self.execute_fn(sqlite_derived_table_dependencies) + self._cached_derived_table_dependencies = (schema_version, dependencies) + return self._cached_derived_table_dependencies[1] + async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] @@ -850,16 +1133,28 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( + "block", + "enqueued_at_ns", "fn", - "task_id", - "loop", - "reply_future", "isolated_connection", + "loop", + "otel_context", + "reply_future", + "task_id", "transaction", ) def __init__( - self, fn, task_id, loop, reply_future, isolated_connection, transaction + self, + fn, + task_id, + loop, + reply_future, + isolated_connection, + transaction, + otel_context, + enqueued_at_ns, + block, ): self.fn = fn self.task_id = task_id @@ -867,6 +1162,9 @@ class WriteTask: self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction + self.otel_context = otel_context + self.enqueued_at_ns = enqueued_at_ns + self.block = block def _deliver_write_result(task, result, exception): @@ -895,7 +1193,7 @@ class QueryInterrupted(Exception): self.params = params def __str__(self): - return "QueryInterrupted: {}".format(self.e) + return f"QueryInterrupted: {self.e}" class MultipleValues(Exception): diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 2f78570b..ee165ae5 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -2,8 +2,8 @@ from datasette import hookimpl from datasette.permissions import Action from datasette.resources import ( DatabaseResource, - TableResource, QueryResource, + TableResource, ) @@ -61,6 +61,12 @@ def register_actions(): description="Create tables", resource_class=DatabaseResource, ), + Action( + name="create-view", + abbr="cv", + description="Create views", + resource_class=DatabaseResource, + ), Action( name="store-query", abbr="sq", @@ -111,6 +117,12 @@ def register_actions(): description="Drop tables", resource_class=TableResource, ), + Action( + name="drop-view", + abbr="dv", + description="Drop views", + resource_class=TableResource, + ), # Query-level actions (child-level) Action( name="view-query", diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index f90a733e..6def3698 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,6 +6,17 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType +_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) + + +def _normalize_http_url(value): + if not isinstance(value, str): + return None + normalized = value.strip() + if not _HTTP_URL_RE.fullmatch(normalized): + return None + return normalized + class UrlColumnType(ColumnType): name = "url" @@ -15,7 +26,10 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - escaped = markupsafe.escape(value.strip()) + normalized = _normalize_http_url(value) + if normalized is None: + return markupsafe.escape(value.strip()) + escaped = markupsafe.escape(normalized) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -23,7 +37,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if not re.match(r"^https?://\S+$", value.strip()): + if _normalize_http_url(value) is None: return "Invalid URL" return None diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index 91c1c5aa..bff5f0a7 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -1,8 +1,9 @@ -from datasette import hookimpl import datetime import os import time +from datasette import hookimpl + def header(key, request): key = key.replace("_", "-").encode("utf-8") diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 6cd46f04..dee5df42 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,18 +17,29 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -# Re-export all hooks and public utilities -from .restrictions import ( - actor_restrictions_sql as actor_restrictions_sql, - restrictions_allow_action as restrictions_allow_action, - ActorRestrictions as ActorRestrictions, -) -from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql +from .defaults import ( + DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, +) +from .defaults import ( + default_action_permissions_sql as default_action_permissions_sql, +) from .defaults import ( # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, - default_action_permissions_sql as default_action_permissions_sql, - default_query_permissions_sql as default_query_permissions_sql, - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, ) +from .defaults import ( + default_query_permissions_sql as default_query_permissions_sql, +) +from .restrictions import ( + ActorRestrictions as ActorRestrictions, +) + +# Re-export all hooks and public utilities +from .restrictions import ( + actor_restrictions_sql as actor_restrictions_sql, +) +from .restrictions import ( + restrictions_allow_action as restrictions_allow_action, +) +from .root import root_user_permissions_sql as root_user_permissions_sql diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index aab87c1c..a4f5a4de 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration. from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from datasette.app import Datasette @@ -55,8 +55,8 @@ class ConfigPermissionProcessor: def __init__( self, - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, ): self.datasette = datasette @@ -74,8 +74,8 @@ class ConfigPermissionProcessor: self.restrictions = actor.get("_r", {}) if actor else {} # Pre-compute restriction info for efficiency - self.restricted_databases: Set[str] = set() - self.restricted_tables: Set[Tuple[str, str]] = set() + self.restricted_databases: set[str] = set() + self.restricted_tables: set[tuple[str, str]] = set() if self.has_restrictions: self.restricted_databases = { @@ -92,16 +92,27 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: + # Resolve identity keys once per action, rather than scanning the + # restriction allowlist for every configured table's allow block. + self.restricted_table_keys = { + (db, self.action_obj.normalize_child(table) if self.action_obj else table) + for db, table in self.restricted_tables + } + + def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: return None + # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as + # integers, but should retain the CLI's boolean 1/0 behavior. + if isinstance(allow_block, int) and allow_block in (0, 1): + return bool(allow_block) return actor_matches_allow(self.actor, allow_block) def is_in_restriction_allowlist( self, - parent: Optional[str], - child: Optional[str], + parent: str | None, + child: str | None, ) -> bool: """Check if resource is allowed by actor restrictions.""" if not self.has_restrictions: @@ -121,8 +132,10 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - table_actions = table_restrictions.get(child, []) - if self.action_checks.intersection(table_actions): + child_key = ( + self.action_obj.normalize_child(child) if self.action_obj else child + ) + if (parent, child_key) in self.restricted_table_keys: return True else: # Parent query should proceed if any child in this database is allowlisted @@ -143,9 +156,9 @@ class ConfigPermissionProcessor: def add_permissions_rule( self, - parent: Optional[str], - child: Optional[str], - permissions_block: Optional[dict], + parent: str | None, + child: str | None, + permissions_block: dict | None, scope_desc: str, ) -> None: """Add a rule from a permissions:{action} block.""" @@ -165,8 +178,8 @@ class ConfigPermissionProcessor: def add_allow_block_rule( self, - parent: Optional[str], - child: Optional[str], + parent: str | None, + child: str | None, allow_block: Any, scope_desc: str, ) -> None: @@ -198,8 +211,8 @@ class ConfigPermissionProcessor: def _add_restriction_gate_denies( self, - parent: Optional[str], - child: Optional[str], + parent: str | None, + child: str | None, is_allowed: bool, scope_desc: str, ) -> None: @@ -231,7 +244,7 @@ class ConfigPermissionProcessor: if db_name == parent: self.collector.add(db_name, table_name, False, reason) - def process(self) -> Optional[PermissionSQL]: + def process(self) -> PermissionSQL | None: """Process all config rules and return combined PermissionSQL.""" self._process_root_permissions() self._process_databases() @@ -421,10 +434,10 @@ class ConfigPermissionProcessor: @hookimpl(specname="permission_resources_sql") async def config_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[List[PermissionSQL]]: +) -> list[PermissionSQL] | None: """ Apply permission rules from datasette.yaml configuration. diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 5bc74425..6f97812b 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -29,29 +29,28 @@ DEFAULT_ALLOW_ACTIONS = frozenset( @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[PermissionSQL]: +) -> PermissionSQL | None: """ Enforce the default_allow_sql setting. When default_allow_sql is false (the default), execute-sql is denied unless explicitly allowed by config or other rules. """ - if action == "execute-sql": - if not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") + if action == "execute-sql" and not datasette.setting("default_allow_sql"): + return PermissionSQL.deny(reason="default_allow_sql is false") return None @hookimpl(specname="permission_resources_sql") async def default_action_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[PermissionSQL]: +) -> PermissionSQL | None: """ Provide default allow rules for standard view/execute actions. @@ -71,10 +70,10 @@ async def default_action_permissions_sql( @hookimpl(specname="permission_resources_sql") async def default_query_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[PermissionSQL]: +) -> PermissionSQL | None: actor_id = actor.get("id") if isinstance(actor, dict) else None if action not in {"view-query", "update-query", "delete-query"}: diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py index 47e03569..5e59b7b4 100644 --- a/datasette/default_permissions/helpers.py +++ b/datasette/default_permissions/helpers.py @@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datasette.permissions import PermissionSQL -def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: +def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: """ Get all name variants for an action (full name and abbreviation). @@ -27,7 +27,7 @@ def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: return variants -def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: +def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool: """Check if an action (or its abbreviation) is in a list.""" return bool(get_action_name_variants(datasette, action).intersection(action_list)) @@ -36,8 +36,8 @@ def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bo class PermissionRow: """A single permission rule row.""" - parent: Optional[str] - child: Optional[str] + parent: str | None + child: str | None allow: bool reason: str @@ -46,14 +46,14 @@ class PermissionRowCollector: """Collects permission rows and converts them to PermissionSQL.""" def __init__(self, prefix: str = "row"): - self.rows: List[PermissionRow] = [] + self.rows: list[PermissionRow] = [] self.prefix = prefix def add( self, - parent: Optional[str], - child: Optional[str], - allow: Optional[bool], + parent: str | None, + child: str | None, + allow: bool | None, reason: str, if_not_none: bool = False, ) -> None: @@ -62,7 +62,7 @@ class PermissionRowCollector: return self.rows.append(PermissionRow(parent, child, allow, reason)) - def to_permission_sql(self) -> Optional[PermissionSQL]: + def to_permission_sql(self) -> PermissionSQL | None: """Convert collected rows to a PermissionSQL object.""" if not self.rows: return None diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index a22cd7e5..d30ebd3f 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -8,7 +8,7 @@ contains allowlists of resources the actor can access. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set, Tuple +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants class ActorRestrictions: """Parsed actor restrictions from the _r key.""" - global_actions: List[str] # _r.a - globally allowed actions + global_actions: list[str] # _r.a - globally allowed actions database_actions: dict # _r.d - {db_name: [actions]} table_actions: dict # _r.r - {db_name: {table: [actions]}} @classmethod - def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: + def from_actor(cls, actor: dict | None) -> ActorRestrictions | None: """Parse restrictions from actor dict. Returns None if no restrictions.""" if not actor: return None @@ -44,11 +44,11 @@ class ActorRestrictions: table_actions=restrictions.get("r", {}), ) - def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: + def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool: """Check if action is in the global allowlist.""" return action_in_list(datasette, action, self.global_actions) - def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: + def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]: """Get database names where this action is allowed.""" allowed = set() for db_name, db_actions in self.database_actions.items(): @@ -57,8 +57,8 @@ class ActorRestrictions: return allowed def get_allowed_tables( - self, datasette: "Datasette", action: str - ) -> Set[Tuple[str, str]]: + self, datasette: Datasette, action: str + ) -> set[tuple[str, str]]: """Get (database, table) pairs where this action is allowed.""" allowed = set() for db_name, tables in self.table_actions.items(): @@ -70,10 +70,10 @@ class ActorRestrictions: @hookimpl(specname="permission_resources_sql") async def actor_restrictions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[List[PermissionSQL]]: +) -> list[PermissionSQL] | None: """ Handle actor restriction-based permission rules. @@ -140,10 +140,10 @@ async def actor_restrictions_sql( def restrictions_allow_action( - datasette: "Datasette", + datasette: Datasette, restrictions: dict, action: str, - resource: Optional[str | Tuple[str, str]], + resource: str | tuple[str, str] | None, ) -> bool: """ Check if restrictions allow the requested action on the requested resource. @@ -185,11 +185,15 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + action_obj = datasette.actions.get(action) + normalize = action_obj.normalize_child if action_obj else lambda name: name + for table_name, table_allowed in ( + restrictions.get("r", {}).get(database, {}).items() + ): + if normalize(table_name) == normalize(table): + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py index 4931f7ff..22d13f65 100644 --- a/datasette/default_permissions/root.py +++ b/datasette/default_permissions/root.py @@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL @hookimpl(specname="permission_resources_sql") async def root_user_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], -) -> Optional[PermissionSQL]: + datasette: Datasette, + actor: dict | None, +) -> PermissionSQL | None: """ Grant root user full permissions when --root flag is used. """ diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py new file mode 100644 index 00000000..11fd4008 --- /dev/null +++ b/datasette/default_permissions/sqlite_statistics.py @@ -0,0 +1,25 @@ +"""Default table-access policy for SQLite optimizer statistics.""" + +import json + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + + +@hookimpl +def permission_resources_sql(action): + if action != "view-table": + return None + return PermissionSQL( + sql=""" + SELECT database_name AS parent, value AS child, 0 AS allow, + 'SQLite statistics tables are denied by default' AS reason + FROM catalog_databases + CROSS JOIN json_each(:sqlite_statistics_names) + """, + params={ + "sqlite_statistics_names": json.dumps( + ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] + ) + }, + ) diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py index 7a359dc6..52daf8a2 100644 --- a/datasette/default_permissions/tokens.py +++ b/datasette/default_permissions/tokens.py @@ -7,7 +7,7 @@ to datasette.verify_token() so all registered handlers are tried. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -17,15 +17,13 @@ from datasette.tokens import SignedTokenHandler @hookimpl -def register_token_handler(datasette: "Datasette"): +def register_token_handler(datasette: Datasette): """Register the default signed token handler.""" return SignedTokenHandler() @hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token( - datasette: "Datasette", request -) -> Optional[dict]: +async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None: """ Authenticate requests using API tokens by delegating to all registered token handlers via datasette.verify_token(). diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py index e41434ef..0f2f32ef 100644 --- a/datasette/default_table_actions.py +++ b/datasette/default_table_actions.py @@ -20,7 +20,7 @@ def table_actions(datasette, actor, database, table, request): "label": "Alter table", "description": "Change columns and primary key for this table.", "attrs": { - "aria-label": "Alter table {}".format(table), + "aria-label": f"Alter table {table}", "data-table-action": "alter-table", }, } diff --git a/datasette/events.py b/datasette/events.py index e8786da9..5f3fd06e 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -1,8 +1,9 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field -from datasette.hookspecs import hookimpl from datetime import datetime, timezone +from datasette.hookspecs import hookimpl + @dataclass class Event(ABC): diff --git a/datasette/extras.py b/datasette/extras.py index 36014185..fb8c2e06 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -5,6 +5,8 @@ from typing import ClassVar from asyncinject import Registry +from datasette.utils.asgi import BadRequest + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -113,6 +115,17 @@ class ExtraRegistry: self._allowed_names[key] = names return names + def validate_requested(self, requested, scope): + """ + Raise BadRequest if any requested extra name is not a public extra + for this scope. Used by data formats such as .json - HTML pages + silently ignore unknown names instead. + """ + allowed = self._allowed_names_for_scope(scope, include_internal=False) + unknown = sorted(name for name in requested if name not in allowed) + if unknown: + raise BadRequest("Unknown _extra: {}".format(", ".join(unknown))) + async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..69ac2c42 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -1,12 +1,13 @@ import json import urllib + from datasette import hookimpl from datasette.database import QueryInterrupted from datasette.utils import ( + detect_json1, escape_sqlite, path_with_added_args, path_with_removed_args, - detect_json1, sqlite3, ) @@ -30,7 +31,7 @@ def load_facet_configs(request, table_config): assert ( len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, facet_config = list(facet_config.items())[0] + type, facet_config = next(iter(facet_config.items())) if isinstance(facet_config, str): facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( @@ -38,7 +39,7 @@ def load_facet_configs(request, table_config): ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): - if key.startswith("_facet"): + if key == "_facet" or key.startswith("_facet_"): # Figure out the facet type if key == "_facet": type = "column" @@ -85,7 +86,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table - self.sql = sql or f"select * from [{table}]" + self.sql = sql or f"select * from {escape_sqlite(table)}" self.params = params or [] self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: @@ -160,18 +161,13 @@ class ColumnFacet(Facet): for column in columns: if column in already_enabled: continue - suggested_facet_sql = """ - with limited as (select * from ({sql}) limit {suggest_consider}) - select {column} as value, count(*) as n from limited + suggested_facet_sql = f""" + with limited as (select * from ({self.sql}) limit {self.suggest_consider}) + select {escape_sqlite(column)} as value, count(*) as n from limited where value is not null group by value - limit {limit} - """.format( - column=escape_sqlite(column), - sql=self.sql, - limit=facet_size + 1, - suggest_consider=self.suggest_consider, - ) + limit {facet_size + 1} + """ distinct_values = None try: distinct_values = await self.ds.execute( @@ -267,11 +263,16 @@ class ColumnFacet(Facet): for row in facet_rows: column_qs = column if column.startswith("_"): - column_qs = "{}__exact".format(column) - selected = (column_qs, str(row["value"])) in qs_pairs + column_qs = f"{column}__exact" + selected_args = { + key: str(row["value"]) + for key in (column_qs, f"{column}__exact") + if (key, str(row["value"])) in qs_pairs + } + selected = bool(selected_args) if selected: toggle_path = path_with_removed_args( - self.request, {column_qs: str(row["value"])} + self.request, selected_args ) else: toggle_path = path_with_added_args( @@ -342,12 +343,12 @@ class ArrayFacet(Facet): for v in await self.ds.execute( self.database, ( - "select {column} from ({sql}) " - "where {column} is not null " - "and {column} != '' " - "and json_array_length({column}) > 0 " + f"select {escape_sqlite(column)} from ({self.sql}) " + f"where {escape_sqlite(column)} is not null " + f"and {escape_sqlite(column)} != '' " + f"and json_array_length({escape_sqlite(column)}) > 0 " "limit 100" - ).format(column=escape_sqlite(column), sql=self.sql), + ), self.params, truncate=False, custom_time_limit=self.ds.setting( @@ -388,14 +389,14 @@ class ArrayFacet(Facet): source = source_and_config["source"] column = config.get("column") or config["simple"] # https://github.com/simonw/datasette/issues/448 - facet_sql = """ - with inner as ({sql}), + facet_sql = f""" + with inner as ({self.sql}), deduped_array_items as ( select distinct j.value, inner.* from - json_each([inner].{col}) j + json_each([inner].{escape_sqlite(column)}) j join inner ) select @@ -406,12 +407,8 @@ class ArrayFacet(Facet): group by value order by - count(*) desc, value limit {limit} - """.format( - col=escape_sqlite(column), - sql=self.sql, - limit=facet_size + 1, - ) + count(*) desc, value limit {facet_size + 1} + """ try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 95cc5f37..0499d086 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,8 +1,12 @@ -from datasette import hookimpl -from datasette.resources import DatabaseResource -from datasette.views.base import DatasetteError -from datasette.utils.asgi import BadRequest import json +import math +from typing import ClassVar + +from datasette import hookimpl +from datasette.resources import DatabaseResource, TableResource +from datasette.utils.asgi import BadRequest +from datasette.views.base import DatasetteError + from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -48,13 +52,20 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which fts_table to use + # Figure out which trusted fts_table to use. Query string parameters can + # repeat this mapping (for backwards compatibility), but must not select + # a different table or primary key. table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = request.args.get("_fts_table") - fts_table = fts_table or table_metadata.get("fts_table") + fts_table = table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) + fts_pk = table_metadata.get("fts_pk", "rowid") + requested_fts_table = request.args.get("_fts_table") + requested_fts_pk = request.args.get("_fts_pk") + if (requested_fts_table and requested_fts_table != fts_table) or ( + requested_fts_pk and requested_fts_pk != fts_pk + ): + raise BadRequest("Invalid _fts_table or _fts_pk") search_args = { key: request.args[key] for key in request.args @@ -72,6 +83,11 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=fts_table), + actor=request.actor, + ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -99,9 +115,9 @@ def search_filters(request, database, table, datasette): fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), match_clause=( - ":search_{}".format(i) + f":search_{i}" if search_mode_raw - else "escape_fts(:search_{})".format(i) + else f"escape_fts(:search_{i})" ), ) ) @@ -132,13 +148,18 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=through_table), + actor=request.actor, + ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - try: - fk_to_us = [ - fk for fk in outgoing_foreign_keys if fk["other_table"] == table - ][0] - except IndexError: + fk_to_us = next( + (fk for fk in outgoing_foreign_keys if fk["other_table"] == table), + None, + ) + if fk_to_us is None: raise DatasetteError( "Invalid _through - could not find corresponding foreign key" ) @@ -182,6 +203,17 @@ class Filter: raise NotImplementedError +def _coerce_numeric_filter_value(value): + try: + return int(value) + except ValueError: + try: + converted = float(value) + except ValueError: + return value + return converted if math.isfinite(converted) else value + + class TemplatedFilter(Filter): def __init__( self, @@ -203,13 +235,17 @@ class TemplatedFilter(Filter): def where_clause(self, table, column, value, param_counter): converted = self.format.format(value) - if self.numeric and converted.isdigit(): - converted = int(converted) + if self.numeric: + converted = _coerce_numeric_filter_value(converted) if self.no_argument: - kwargs = {"c": column} + kwargs = {"c": _quote_sqlite_identifier(column)} converted = None else: - kwargs = {"c": column, "p": f"p{param_counter}", "t": table} + kwargs = { + "c": _quote_sqlite_identifier(column), + "p": f"p{param_counter}", + "t": _quote_sqlite_identifier(table), + } return self.sql_template.format(**kwargs), converted def human_clause(self, column, value): @@ -223,6 +259,14 @@ class TemplatedFilter(Filter): return template.format(c=column, v=value) +def _quote_sqlite_identifier(identifier): + # Preserve the historic always-quoted SQL generated by TemplatedFilter. + escaped = escape_sqlite(identifier) + if escaped == identifier: + return f'"{identifier}"' + return escaped + + class InFilter(Filter): key = "in" display = "in" @@ -264,56 +308,56 @@ class Filters: TemplatedFilter( "exact", "=", - '"{c}" = :{p}', + "{c} = :{p}", lambda c, v: "{c} = {v}" if v.isdigit() else '{c} = "{v}"', ), TemplatedFilter( "not", "!=", - '"{c}" != :{p}', + "{c} != :{p}", lambda c, v: "{c} != {v}" if v.isdigit() else '{c} != "{v}"', ), TemplatedFilter( "contains", "contains", - '"{c}" like :{p}', + "{c} like :{p}", '{c} contains "{v}"', format="%{}%", ), TemplatedFilter( "notcontains", "does not contain", - '"{c}" not like :{p}', + "{c} not like :{p}", '{c} does not contain "{v}"', format="%{}%", ), TemplatedFilter( "endswith", "ends with", - '"{c}" like :{p}', + "{c} like :{p}", '{c} ends with "{v}"', format="%{}", ), TemplatedFilter( "startswith", "starts with", - '"{c}" like :{p}', + "{c} like :{p}", '{c} starts with "{v}"', format="{}%", ), - TemplatedFilter("gt", ">", '"{c}" > :{p}', "{c} > {v}", numeric=True), + TemplatedFilter("gt", ">", "{c} > :{p}", "{c} > {v}", numeric=True), TemplatedFilter( - "gte", "\u2265", '"{c}" >= :{p}', "{c} \u2265 {v}", numeric=True + "gte", "\u2265", "{c} >= :{p}", "{c} \u2265 {v}", numeric=True ), - TemplatedFilter("lt", "<", '"{c}" < :{p}', "{c} < {v}", numeric=True), + TemplatedFilter("lt", "<", "{c} < :{p}", "{c} < {v}", numeric=True), TemplatedFilter( - "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True + "lte", "\u2264", "{c} <= :{p}", "{c} \u2264 {v}", numeric=True ), - TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'), + TemplatedFilter("like", "like", "{c} like :{p}", '{c} like "{v}"'), TemplatedFilter( - "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"' + "notlike", "not like", "{c} not like :{p}", '{c} not like "{v}"' ), - TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'), + TemplatedFilter("glob", "glob", "{c} glob :{p}", '{c} glob "{v}"'), InFilter(), NotInFilter(), ] @@ -322,13 +366,13 @@ class Filters: TemplatedFilter( "arraycontains", "array contains", - """:{p} in (select value from json_each([{t}].[{c}]))""", + """:{p} in (select value from json_each({t}.{c}))""", '{c} contains "{v}"', ), TemplatedFilter( "arraynotcontains", "array does not contain", - """:{p} not in (select value from json_each([{t}].[{c}]))""", + """:{p} not in (select value from json_each({t}.{c}))""", '{c} does not contain "{v}"', ), ] @@ -336,36 +380,34 @@ class Filters: else [] ) + [ + TemplatedFilter("date", "date", "date({c}) = :{p}", '"{c}" is on date {v}'), TemplatedFilter( - "date", "date", 'date("{c}") = :{p}', '"{c}" is on date {v}' - ), - TemplatedFilter( - "isnull", "is null", '"{c}" is null', "{c} is null", no_argument=True + "isnull", "is null", "{c} is null", "{c} is null", no_argument=True ), TemplatedFilter( "notnull", "is not null", - '"{c}" is not null', + "{c} is not null", "{c} is not null", no_argument=True, ), TemplatedFilter( "isblank", "is blank", - '("{c}" is null or "{c}" = "")', + "({c} is null or {c} = '')", "{c} is blank", no_argument=True, ), TemplatedFilter( "notblank", "is not blank", - '("{c}" is not null and "{c}" != "")', + "({c} is not null and {c} != '')", "{c} is not blank", no_argument=True, ), ] ) - _filters_by_key = {f.key: f for f in _filters} + _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters} def __init__(self, pairs): self.pairs = pairs diff --git a/datasette/fixtures.py b/datasette/fixtures.py index 7c85e16a..049e35ed 100644 --- a/datasette/fixtures.py +++ b/datasette/fixtures.py @@ -1,9 +1,10 @@ -from datasette.utils.sqlite import sqlite3 -from datasette.utils import documented import itertools import random import string +from datasette.utils import documented +from datasette.utils.sqlite import sqlite3 + __all__ = [ "EXTRA_DATABASE_SQL", "TABLES", @@ -346,9 +347,7 @@ CREATE VIEW searchable_view_configured_by_metadata AS + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + "\n".join( [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) + f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");' for a, b, c, content in generate_compound_rows(1001) ] ) diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 41c48396..67bf0d8b 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,9 +1,20 @@ -from datasette import hookimpl, Response +from datasette import Response, hookimpl + +from .utils import add_cors_headers @hookimpl(trylast=True) def forbidden(datasette, request, message): async def inner(): + if ( + request.path.split("?")[0].endswith(".json") + or "application/json" in (request.headers.get("accept") or "") + or request.headers.get("content-type") == "application/json" + ): + headers = {} + if datasette.cors: + add_cors_headers(headers) + return Response.error(message, 403, headers=headers) return Response.html( await datasette.render_template( "error.html", diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 2b311644..ef6c7b7e 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,16 +1,21 @@ -from datasette import hookimpl, Response -from .utils import add_cors_headers +import traceback + +from markupsafe import Markup + +from datasette import Response, hookimpl + +from .utils import add_cors_headers, error_body from .utils.asgi import ( Base400, ) from .views.base import DatasetteError -from markupsafe import Markup -import traceback +# Debugger imports are deliberate - they back the "pdb" setting, which drops +# into a debugger on unhandled exceptions try: - import ipdb as pdb + import ipdb as pdb # noqa: T100 except ImportError: - import pdb + import pdb # noqa: T100 try: import rich @@ -28,6 +33,7 @@ def handle_exception(datasette, request, exception): rich.get_console().print_exception(show_locals=True) title = None + plain_message = None if isinstance(exception, Base400): status = exception.status info = {} @@ -36,6 +42,7 @@ def handle_exception(datasette, request, exception): status = exception.status info = exception.error_dict message = exception.message + plain_message = exception.plain_message if exception.message_is_html: message = Markup(message) title = exception.title @@ -45,6 +52,17 @@ def handle_exception(datasette, request, exception): message = str(exception) traceback.print_exc() templates = [f"{status}.html", "error.html"] + headers = {} + if datasette.cors: + add_cors_headers(headers) + if request.path.split("?")[0].endswith(".json"): + body = dict(info) + body.update(error_body(plain_message or message, status)) + return Response.json(body, status=status, headers=headers) + if request.path.split("?")[0].endswith(".csv"): + return Response.text( + plain_message or message, status=status, headers=headers + ) info.update( { "ok": False, @@ -53,24 +71,18 @@ def handle_exception(datasette, request, exception): "title": title, } ) - headers = {} - if datasette.cors: - add_cors_headers(headers) - if request.path.split("?")[0].endswith(".json"): - return Response.json(info, status=status, headers=headers) - else: - environment = datasette.get_jinja_environment(request) - template = environment.select_template(templates) - return Response.html( - await template.render_async( - dict( - info, - urls=datasette.urls, - menu_links=lambda: [], - ) - ), - status=status, - headers=headers, - ) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) + return Response.html( + await template.render_async( + dict( + info, + urls=datasette.urls, + menu_links=list, + ) + ), + status=status, + headers=headers, + ) return inner diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 7c56f882..49d8e8ea 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -1,5 +1,4 @@ -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("datasette") hookimpl = HookimplMarker("datasette") @@ -10,6 +9,11 @@ def startup(datasette): """Fires directly after Datasette first starts running""" +@hookspec +def shutdown(datasette): + """Called once when the Datasette server is shutting down""" + + @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" @@ -46,7 +50,7 @@ def extra_body_script( def extra_template_vars( template, database, table, columns, view_name, request, datasette ): - """Extra template variables to be made available to the template - can return dict or callable or awaitable""" + """Extra template variables to be made available to the template - can return dict, None, callable or awaitable""" @hookspec diff --git a/datasette/inspect.py b/datasette/inspect.py index 5e681e03..b126ce5c 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -1,13 +1,13 @@ import hashlib from .utils import ( - detect_spatialite, detect_fts, detect_primary_keys, + detect_spatialite, escape_sqlite, get_all_foreign_keys, - table_columns, sqlite3, + table_columns, ) HASH_BLOCK_SIZE = 1024 * 1024 @@ -95,10 +95,10 @@ def inspect_tables(conn, database_metadata): """) ] - for t in tables.keys(): + for t, table_info in tables.items(): for hidden_table in hidden_tables: if t == hidden_table or t.startswith(hidden_table): - tables[t]["hidden"] = True + table_info["hidden"] = True continue return tables diff --git a/datasette/jump.py b/datasette/jump.py index d138e827..d70d33df 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -21,7 +21,7 @@ class JumpSQL: search_text: str | None = None, display_name: str | None = None, item_type: str = "menu", - ) -> "JumpSQL": + ) -> JumpSQL: if search_text is None: search_text = " ".join( text for text in (label, display_name, description) if text is not None diff --git a/datasette/permissions.py b/datasette/permissions.py index 786dc026..2d242560 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,7 +1,11 @@ +import contextvars from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple -import contextvars + +_SQLITE_IDENTIFIER_CASE = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( @@ -49,6 +53,15 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables + case_insensitive_child: bool = False + + @classmethod + def normalize_child(cls, child: str | None) -> str | None: + """Return a comparison key without changing the resource's display name.""" + if cls.case_insensitive_child and child is not None: + # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. + return child.translate(_SQLITE_IDENTIFIER_CASE) + return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -72,8 +85,8 @@ class Resource(ABC): ) def __repr__(self) -> str: - return "{}(parent={!r}, child={!r})".format( - self.__class__.__name__, self.parent, self.child + return ( + f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})" ) @property @@ -129,7 +142,6 @@ class Resource(ABC): Must return two columns: parent, child """ - pass class AllowedResource(NamedTuple): @@ -147,6 +159,11 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed + def normalize_child(self, child: str | None) -> str | None: + if self.resource_class is None: + return child + return self.resource_class.normalize_child(child) + @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index ae2cb17d..6a4d7da7 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,20 +1,14 @@ import importlib +import importlib.metadata as importlib_metadata +import importlib.resources as importlib_resources import os -import pluggy -from pprint import pprint import sys +from pprint import pprint + +import pluggy + from . import hookspecs -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources -if sys.version_info >= (3, 10): - import importlib.metadata as importlib_metadata -else: - import importlib_metadata - - DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -24,6 +18,7 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", + "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", @@ -85,7 +80,7 @@ if DATASETTE_LOAD_PLUGINS is not None: # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) except importlib_metadata.PackageNotFoundError: - sys.stderr.write("Plugin {} could not be found\n".format(package_name)) + sys.stderr.write(f"Plugin {package_name} could not be found\n") # Load default plugins diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 63d22fe8..9ace865b 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,15 +1,17 @@ -from datasette import hookimpl -import click import json import os import re from subprocess import CalledProcessError, check_call, check_output +import click + +from datasette import hookimpl + +from ..utils import temporary_docker_directory from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) -from ..utils import temporary_docker_directory @hookimpl @@ -219,7 +221,7 @@ def publish_subcommand(publish): check_call( "gcloud builds submit --tag {}{}".format( - image_id, " --timeout {}".format(timeout) if timeout else "" + image_id, f" --timeout {timeout}" if timeout else "" ), shell=True, ) @@ -231,7 +233,7 @@ def publish_subcommand(publish): ("--min-instances", min_instances), ): if value is not None: - extra_deploy_options.append("{} {}".format(option, value)) + extra_deploy_options.append(f"{option} {value}") check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, @@ -258,24 +260,16 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi ) from exc describe_cmd = ( - "gcloud artifacts repositories describe {repo} --project {project} " - "--location {location} --quiet" - ).format( - repo=artifact_repository, - project=artifact_project, - location=artifact_region, + f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} " + f"--location {artifact_region} --quiet" ) try: check_call(describe_cmd, shell=True) return except CalledProcessError: create_cmd = ( - "gcloud artifacts repositories create {repo} --repository-format=docker " - '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' - ).format( - repo=artifact_repository, - location=artifact_region, - project=artifact_project, + f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker " + f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet' ) try: check_call(create_cmd, shell=True) diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 29665eb3..27dfd4bf 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -1,9 +1,11 @@ -from ..utils import StaticMount -import click import os import shutil import sys +import click + +from ..utils import StaticMount + def add_common_publish_arguments_and_options(subcommand): for decorator in reversed( @@ -76,9 +78,7 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): """Exit (with error message) if ``binary` isn't installed""" if not shutil.which(binary): click.secho( - "Publishing to {publish_target} requires {binary} to be installed and configured".format( - publish_target=publish_target, binary=binary - ), + f"Publishing to {publish_target} requires {binary} to be installed and configured", bg="red", fg="white", bold=True, diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index f576a346..b0290833 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -1,19 +1,21 @@ -from contextlib import contextmanager -from datasette import hookimpl -import click import json import os import pathlib import shlex import shutil -from subprocess import call, check_output import tempfile +from contextlib import contextmanager +from subprocess import call, check_output + +import click + +from datasette import hookimpl +from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) -from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata @hookimpl @@ -234,7 +236,7 @@ def temporary_heroku_directory( extras.extend(["--static", f"{mount_point}:{mount_point}"]) quoted_files = " ".join( - ["-i {}".format(shlex.quote(file_name)) for file_name in file_names] + [f"-i {shlex.quote(file_name)}" for file_name in file_names] ) procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) diff --git a/datasette/renderer.py b/datasette/renderer.py index f40e3dbb..0e01f52f 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,11 +1,13 @@ import json + from datasette.extras import extra_names_from_request from datasette.utils import ( - value_as_boolean, - remove_infinites, CustomJSONEncoder, + error_body, path_from_row_pks, + remove_infinites, sqlite3, + value_as_boolean, ) from datasette.utils.asgi import Response @@ -52,8 +54,7 @@ def json_renderer(request, args, data, error, truncated=None): if error: shape = "objects" status_code = 400 - data["error"] = error - data["ok"] = False + data.update(error_body(error, status_code)) if truncated is not None: data["truncated"] = truncated @@ -87,7 +88,8 @@ def json_renderer(request, args, data, error, truncated=None): object_rows[pk_string] = row data = object_rows if shape_error: - data = {"ok": False, "error": shape_error} + status_code = 400 + data = error_body(shape_error, status_code) elif shape == "array": data = data["rows"] @@ -100,12 +102,7 @@ def json_renderer(request, args, data, error, truncated=None): data["rows"] = [list(row.values()) for row in data["rows"]] else: status_code = 400 - data = { - "ok": False, - "error": f"Invalid _shape: {shape}", - "status": 400, - "title": None, - } + data = error_body(f"Invalid _shape: {shape}", status_code) # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 diff --git a/datasette/resources.py b/datasette/resources.py index ee2e6d98..29bf7b1e 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,6 +25,7 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource + case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/app.css b/datasette/static/app.css index ce800f61..0297371c 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1,3 +1,144 @@ +/* Shared modal styles. */ +datasette-modal { + display: contents; +} + +dialog.datasette-modal { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(520px, calc(100vw - 32px)); + max-width: 95vw; + max-height: calc(100dvh - 32px); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.datasette-modal[open] { + display: flex; + flex-direction: column; +} + +dialog.datasette-modal::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +@keyframes datasette-modal-slide-in { + from { opacity: 0; transform: translateY(-20px) scale(0.95); } + to { opacity: 1; transform: translateY(0) scale(1); } +} + +@keyframes datasette-modal-fade-in { + from { opacity: 0; } + to { opacity: 1; } +} + +:where(.datasette-modal) .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; +} + +:where(.datasette-modal) .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +:where(.datasette-modal) .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; +} + +:where(.datasette-modal) .modal-body { + min-height: 0; + overflow: auto; + padding: 16px 24px 24px; +} + +:where(.datasette-modal) .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +:where(.datasette-modal) .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +:where(.datasette-modal) .modal-btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +:where(.datasette-modal) .modal-btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +:where(.datasette-modal) .modal-btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +:where(.datasette-modal) .modal-btn-primary { + background: var(--accent); + color: #fff; +} + +:where(.datasette-modal) .modal-btn-primary:hover { + background: #1949b8; +} + +:where(.datasette-modal) .modal-btn:disabled { + opacity: 0.65; + cursor: wait; +} + +@media (prefers-reduced-motion: reduce) { + dialog.datasette-modal, + dialog.datasette-modal::backdrop { + animation: none; + } +} + /* Reset and Page Setup ==================================================== */ /* Reset from http://meyerweb.com/eric/tools/css/reset/ @@ -63,7 +204,7 @@ em { } /* end reset */ -/* Modal CSS variables (shared by web components via Shadow DOM) */ +/* Shared modal CSS variables */ :root { --modal-backdrop-bg: rgba(0, 0, 0, 0.5); --modal-backdrop-blur: blur(4px); @@ -216,6 +357,49 @@ a:active { text-decoration: underline; } +.table-summary .count-all ~ .table-summary-description { + margin-left: 0.5rem; +} + +.table-summary .count-error:not(:empty) { + display: block; + margin-top: 0.25rem; + font-size: 0.875rem; + font-weight: 400; + line-height: 1.5; +} + +button.count-all { + background: none; + border: none; + padding: 3px 0; + margin-left: 0.25rem; + color: #276890; + font-family: inherit; + font-size: 0.8125rem; + font-weight: 400; + line-height: 1.5; + cursor: pointer; +} + +button.count-all:hover, +button.count-all:focus-visible { + text-decoration: underline; +} + +button.count-all:disabled { + color: #596478; + cursor: wait; +} + +@media (pointer: coarse) { + button.count-all { + min-height: 44px; + padding-left: 7px; + padding-right: 7px; + } +} + button.button-as-link { background: none; border: none; @@ -938,84 +1122,552 @@ p.zero-results { display: none; } -@keyframes datasette-modal-slide-in { - from { - opacity: 0; - transform: translateY(-20px) scale(0.95); +/* navigation-search */ +navigation-search { + display: contents; +} + +navigation-search dialog.datasette-modal { + max-width: 90vw; + width: 600px; + max-height: 80vh; +} + +navigation-search .search-container { + display: flex; + flex-direction: column; +} + +navigation-search .search-input-wrapper { + padding: 1.25rem; + border-bottom: 1px solid #e5e7eb; + display: flex; + gap: 0.5rem; + align-items: center; +} + +navigation-search .search-input { + width: 100%; + flex: 1; + min-width: 0; + padding: 0.75rem 1rem; + font-size: 1rem; + border: 2px solid #e5e7eb; + border-radius: 0.5rem; + outline: none; + transition: border-color 0.2s; + box-sizing: border-box; +} + +navigation-search .search-input:focus { + border-color: #2563eb; +} + +navigation-search .close-search { + background: transparent; + border: 1px solid transparent; + border-radius: 0.375rem; + color: #4b5563; + cursor: pointer; + flex: 0 0 auto; + font: inherit; + font-size: 1.5rem; + height: 2.75rem; + line-height: 1; + width: 2.75rem; +} + +navigation-search .close-search:hover, +navigation-search .close-search:focus { + background-color: #f3f4f6; + border-color: #d1d5db; +} + +navigation-search .results-container { + box-sizing: content-box; + height: calc(80vh - 180px); + padding: 0.5rem; +} + +navigation-search .results-list:empty { + display: none; +} + +navigation-search .result-item { + padding: 0.875rem 1rem; + cursor: pointer; + border-radius: 0.5rem; + transition: background-color 0.15s; + display: flex; + align-items: center; + gap: 0.75rem; +} + +navigation-search .result-item:hover { + background-color: #f3f4f6; +} + +navigation-search .result-item.selected { + background-color: #dbeafe; +} + +navigation-search .result-item > div { + flex: 1; + min-width: 0; +} + +navigation-search .jump-start-content { + border-bottom: 1px solid #e5e7eb; + margin-bottom: 0.5rem; + padding: 0.5rem 0.5rem 1rem; +} + +navigation-search .jump-start-content:empty { + display: none; +} + +navigation-search .result-name { + font-weight: 500; + color: #111827; +} + +navigation-search .result-label { + font-size: 0.875rem; + color: #4b5563; +} + +navigation-search .result-type { + color: #4b5563; + font-size: 0.75rem; + font-weight: 600; + text-transform: uppercase; +} + +navigation-search .result-url { + font-size: 0.875rem; + color: #6b7280; +} + +navigation-search .result-description { + color: #374151; + display: -webkit-box; + font-size: 0.8125rem; + line-height: 1.35; + margin-top: 0.35rem; + overflow: hidden; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; +} + +navigation-search .results-heading { + color: #4b5563; + font-size: 0.75rem; + font-weight: 600; + letter-spacing: 0; + padding: 0.5rem 1rem 0.25rem; + text-transform: uppercase; +} + +navigation-search .recent-actions { + padding: 0.25rem 1rem 0.75rem; +} + +navigation-search .clear-recent { + background: transparent; + border: 0; + color: #2563eb; + cursor: pointer; + font: inherit; + font-size: 0.875rem; + padding: 0; +} + +navigation-search .clear-recent:hover { + text-decoration: underline; +} + +navigation-search .no-results { + padding: 2rem; + text-align: center; + color: #6b7280; +} + +navigation-search .hint-text { + padding: 0.75rem 1.25rem; + font-size: 0.875rem; + color: #6b7280; + border-top: 1px solid #e5e7eb; + display: flex; + gap: 1rem; + flex-wrap: wrap; +} + +navigation-search .hint-text kbd { + background: #f3f4f6; + padding: 0.125rem 0.375rem; + border-radius: 0.25rem; + font-size: 0.75rem; + border: 1px solid #d1d5db; + font-family: monospace; +} + +navigation-search .visually-hidden { + border: 0; + clip: rect(0 0 0 0); + height: 1px; + margin: -1px; + overflow: hidden; + padding: 0; + position: absolute; + white-space: nowrap; + width: 1px; +} + +@media (max-width: 640px) { + navigation-search dialog.datasette-modal { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; } - to { - opacity: 1; - transform: translateY(0) scale(1); + + navigation-search .search-input-wrapper { + padding: 1rem; + } + + navigation-search .search-input { + font-size: 16px; + } + + navigation-search .result-item { + padding: 1rem 0.75rem; + } + + navigation-search .hint-text { + font-size: 0.8rem; + padding: 0.5rem 1rem; } } -@keyframes datasette-modal-fade-in { - from { opacity: 0; } - to { opacity: 1; } -} -dialog.mobile-column-actions-dialog { +/* column-chooser */ +column-chooser { + display: contents; --ink: #0f0f0f; --paper: #eef6ff; --muted: #6b6b6b; --rule: #d8e6f5; --accent: #1a56db; + --accent-light: #e8effd; --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); +} + +column-chooser * { + box-sizing: border-box; + margin: 0; padding: 0; - margin: auto; - width: min(420px, calc(100vw - 32px)); - max-width: 95vw; +} + +column-chooser dialog.datasette-modal { + width: 100%; + max-width: 420px; max-height: min(640px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); + -webkit-user-select: none; + -webkit-touch-callout: none; + -webkit-tap-highlight-color: transparent; } -dialog.mobile-column-actions-dialog[open] { - display: flex; - flex-direction: column; +column-chooser dialog.datasette-modal[open] { + height: min(640px, calc(100vh - 32px)); } -dialog.mobile-column-actions-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.mobile-column-actions-dialog .modal-header { +column-chooser .modal-header { padding: 20px 24px 16px; + justify-content: space-between; +} + +column-chooser .list-toolbar { + padding: 6px 24px; border-bottom: 1px solid var(--rule); display: flex; - align-items: center; - justify-content: space-between; gap: 12px; flex-shrink: 0; } -.mobile-column-actions-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); +column-chooser .list-toolbar button { + background: var(--accent-light); + border: 1px solid var(--rule); + border-radius: 4px; + font-family: inherit; + font-size: 0.75rem; + color: var(--accent); + cursor: pointer; + padding: 3px 10px; + transition: + background 0.12s, + color 0.12s; } -.mobile-column-actions-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; +column-chooser .list-toolbar button:hover { + background: var(--accent); + color: white; +} + +column-chooser .list-wrap { + flex: 1; + padding: 0; + overflow-x: hidden; + position: relative; + overscroll-behavior: contain; + -webkit-overflow-scrolling: touch; +} + +column-chooser .list-wrap::before, +column-chooser .list-wrap::after { + content: ""; + position: sticky; + display: block; + left: 0; + right: 0; + height: 20px; + pointer-events: none; + z-index: 5; + transition: opacity 0.2s; +} + +column-chooser .list-wrap::before { + top: 0; + background: linear-gradient( + to bottom, + rgba(255, 255, 255, 0.9), + transparent + ); +} + +column-chooser .list-wrap::after { + bottom: 0; + background: linear-gradient(to top, rgba(255, 255, 255, 0.9), transparent); + margin-top: -20px; +} + +column-chooser .scroll-zone { + position: absolute; + left: 0; + right: 0; + height: 72px; + pointer-events: none; + z-index: 10; +} + +column-chooser .scroll-zone-top { + top: 0; +} + +column-chooser .scroll-zone-bot { + bottom: 0; +} + +column-chooser .drag-list { + list-style: none; + padding: 4px 0; +} + +column-chooser .drag-item { + display: flex; + align-items: center; + background: white; + border-bottom: 1px solid var(--rule); + user-select: none; + -webkit-user-select: none; + -webkit-touch-callout: none; + position: relative; + transition: background 0.08s; +} + +column-chooser .drag-item:last-child { + border-bottom: none; +} + +column-chooser .drag-handle { + display: flex; + align-items: center; + justify-content: center; + width: 48px; + height: 48px; + flex-shrink: 0; + cursor: grab; + color: #c8c4bc; + touch-action: none; + transition: color 0.15s; +} + +column-chooser .drag-handle:hover { + color: var(--accent); +} + +column-chooser .drag-handle svg { + pointer-events: none; + display: block; +} + +column-chooser .drag-item-content { + display: flex; + align-items: center; + flex: 1; + min-width: 0; + cursor: pointer; +} + +column-chooser .drag-item-check { + display: flex; + align-items: center; + width: 32px; + height: 48px; + flex-shrink: 0; +} + +column-chooser .drag-item-check input[type="checkbox"] { + width: 16px; + height: 16px; + accent-color: var(--accent); + cursor: pointer; +} + +column-chooser .drag-item-label { + flex: 1; + font-size: 0.9rem; + line-height: 48px; + padding-right: 16px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + cursor: default; +} + +column-chooser .drag-item.is-dragging { + opacity: 0; +} + +column-chooser .drop-indicator { + position: absolute; + left: 48px; + right: 0; + height: 2px; + background: var(--accent); + border-radius: 99px; + pointer-events: none; + z-index: 20; + display: none; +} + +column-chooser .drop-indicator.top { + top: -1px; + display: block; +} + +column-chooser .drop-indicator.bottom { + bottom: -1px; + display: block; +} + +column-chooser .drag-ghost { + position: fixed; + pointer-events: none; + z-index: 9999; + background: white; + border-radius: 6px; + box-shadow: + 0 8px 32px rgba(0, 0, 0, 0.18), + 0 2px 8px rgba(0, 0, 0, 0.1); + display: flex; + align-items: center; + border: 1.5px solid var(--accent-light); + opacity: 0.97; + will-change: transform; + font-family: + system-ui, + -apple-system, + sans-serif; +} + +column-chooser .scroll-pulse { + position: absolute; + left: 50%; + transform: translateX(-50%); + width: 32px; + height: 32px; + border-radius: 50%; + background: var(--accent); + opacity: 0; + pointer-events: none; + z-index: 10; + transition: opacity 0.15s; +} + +column-chooser .scroll-pulse.top { + top: 8px; +} + +column-chooser .scroll-pulse.bot { + bottom: 8px; +} + +column-chooser .scroll-pulse.active { + opacity: 0.18; + animation: column-chooser-pulse 0.8s ease-in-out infinite; +} + +@keyframes column-chooser-pulse { + 0%, + 100% { + transform: translateX(-50%) scale(1); + opacity: 0.18; + } + 50% { + transform: translateX(-50%) scale(1.5); + opacity: 0.07; + } +} + +column-chooser .modal-btn-primary { + color: white; +} + +column-chooser .modal-btn-primary:hover { + background: #1448c0; +} + +column-chooser .list-wrap::-webkit-scrollbar { + width: 5px; +} + +column-chooser .list-wrap::-webkit-scrollbar-track { + background: transparent; +} + +column-chooser .list-wrap::-webkit-scrollbar-thumb { + background: var(--rule); + border-radius: 99px; +} + +column-chooser input, +column-chooser textarea { + -webkit-user-select: auto; + user-select: auto; +} + +dialog.mobile-column-actions-dialog { + width: min(420px, calc(100vw - 32px)); + max-height: min(640px, calc(100vh - 32px)); +} + +.mobile-column-actions-dialog .modal-header { + padding: 20px 24px 16px; + justify-content: space-between; } .mobile-column-actions-dialog .list-wrap { flex: 1 1 auto; - min-height: 0; - overflow-y: auto; + padding: 0; overflow-x: hidden; position: relative; overscroll-behavior: contain; @@ -1142,102 +1794,12 @@ dialog.mobile-column-actions-dialog::backdrop { font-size: 0.85em; } -.mobile-column-actions-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.mobile-column-actions-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.mobile-column-actions-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.mobile-column-actions-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.mobile-column-actions-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - dialog.set-column-type-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(720px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.set-column-type-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.set-column-type-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .set-column-type-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.set-column-type-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.set-column-type-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; } .set-column-type-status, @@ -1259,8 +1821,6 @@ dialog.set-column-type-dialog::backdrop { } .set-column-type-options { - padding: 16px 24px 24px; - overflow-y: auto; display: grid; gap: 12px; } @@ -1302,60 +1862,6 @@ dialog.set-column-type-dialog::backdrop { font-size: 0.9rem; } -.set-column-type-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.set-column-type-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.set-column-type-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.set-column-type-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.set-column-type-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.set-column-type-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.set-column-type-dialog .btn-primary:hover { - background: #1949b8; -} - -.set-column-type-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - .row-mutation-status { margin: 0 0 0.75rem; padding: 8px 10px; @@ -1389,46 +1895,11 @@ button.table-insert-row svg { } dialog.row-delete-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(440px, calc(100vw - 32px)); - max-width: 95vw; - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.row-delete-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.row-delete-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .row-delete-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; justify-content: flex-start; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .row-delete-dialog .modal-title { @@ -1437,9 +1908,6 @@ dialog.row-delete-dialog::backdrop { gap: 0.35rem; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .row-delete-message, @@ -1471,94 +1939,12 @@ dialog.row-delete-dialog::backdrop { .row-delete-dialog .modal-footer { padding: 18px 20px 14px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); margin-top: 18px; } -.row-delete-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.row-delete-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.row-delete-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.row-delete-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.row-delete-dialog .btn-primary:hover { - background: #1949b8; -} - -.row-delete-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - dialog.row-edit-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(720px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.row-edit-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.row-edit-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.row-edit-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .row-edit-dialog .modal-title { @@ -1567,9 +1953,6 @@ dialog.row-edit-dialog::backdrop { gap: 0.35rem; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .row-edit-dialog .modal-title .row-dialog-action, @@ -1637,8 +2020,11 @@ dialog.row-edit-dialog::backdrop { .row-edit-fields { display: grid; gap: 14px; - padding: 16px 24px 24px; - overflow-y: auto; +} + +.row-edit-fields[hidden], +.row-edit-bulk[hidden] { + display: none; } .row-edit-field { @@ -1700,6 +2086,118 @@ textarea.row-edit-input { background: var(--paper); } +.row-edit-binary-control { + display: grid; + gap: 8px; + box-sizing: border-box; + width: 100%; + min-width: 0; + border: 1px solid var(--rule); + border-radius: 5px; + padding: 10px; + background: #fff; +} + +.row-edit-binary-control:focus { + border-color: var(--accent); + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.row-edit-binary-preview[hidden] { + display: none; +} + +.row-edit-binary-preview img { + display: block; + max-width: min(240px, 100%); + max-height: 180px; + border: 1px solid var(--rule); + border-radius: 4px; + background: var(--paper); +} + +.row-edit-binary-status { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 8px; + min-width: 0; +} + +.row-edit-binary-size { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.86rem; +} + +.row-edit-binary-name { + color: var(--muted); + font-size: 0.82rem; + overflow-wrap: anywhere; +} + +.row-edit-binary-name[hidden] { + display: none; +} + +.row-edit-binary-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.row-edit-binary-file-button, +.row-edit-binary-clear { + appearance: none; + border: 1px solid var(--rule); + border-radius: 4px; + background: #fff; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.78rem; + line-height: 1.2; + padding: 6px 8px; +} + +.row-edit-binary-file-button:hover, +.row-edit-binary-file-button:focus-within, +.row-edit-binary-clear:hover, +.row-edit-binary-clear:focus { + background: #f8fafc; +} + +.row-edit-binary-file-button:focus-within, +.row-edit-binary-clear:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.row-edit-binary-file-button input[type="file"] { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; + overflow: hidden; +} + +.row-edit-binary-clear[hidden] { + display: none; +} + +.row-edit-binary-drop-target { + border: 1px dashed var(--rule); + border-radius: 4px; + padding: 7px 8px; + color: var(--muted); + font-size: 0.78rem; +} + +.row-edit-binary-dragover .row-edit-binary-drop-target { + border-color: var(--accent); + background: var(--paper); + color: var(--ink); +} + .row-edit-default { display: grid; grid-template-columns: minmax(0, 1fr) 7.25rem; @@ -1798,6 +2296,205 @@ textarea.row-edit-input { margin: 0; } +.row-edit-bulk { + display: grid; + gap: 8px; +} + +.row-edit-bulk-editor { + display: grid; + gap: 8px; +} + +.row-edit-bulk-editor[hidden] { + display: none; +} + +.row-edit-bulk-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px; + justify-content: flex-start; +} + +.row-edit-bulk-actions .modal-btn { + padding-left: 12px; + padding-right: 12px; +} + +.row-edit-bulk-conflict { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 8px 12px; + align-items: start; +} + +.row-edit-bulk-conflict[hidden] { + display: none; +} + +.row-edit-bulk-conflict-label { + color: var(--ink); + font-size: 0.82rem; + padding-top: 8px; +} + +.row-edit-bulk-conflict-control { + display: grid; + gap: 4px; +} + +.row-edit-bulk-conflict-help { + color: var(--muted); + font-size: 0.78rem; + margin: 0; +} + +.row-edit-copy-template-label-narrow { + display: none; +} + +.row-edit-bulk-template-note { + color: var(--muted); + font-size: 0.82rem; +} + +.row-edit-bulk-template-note-narrow { + display: none; +} + +.row-edit-bulk-textarea { + min-height: 16rem; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.45; +} + +.row-edit-bulk-textarea.row-edit-bulk-drop-target { + border-color: var(--accent); + background: #f8fbff; + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.row-edit-bulk-note { + color: var(--muted); + font-size: 0.82rem; + margin: 0; +} + +.row-edit-bulk-note label, +.row-edit-bulk-note .button-as-link { + font: inherit; +} + +@media (max-width: 640px) { + .row-edit-copy-template-label-wide { + display: none; + } + + .row-edit-copy-template-label-narrow { + display: inline; + } + + .row-edit-bulk-template-note-wide { + display: none; + } + + .row-edit-bulk-template-note-narrow { + display: inline; + } +} + +.row-edit-bulk-preview { + display: grid; + gap: 8px; + margin-top: 8px; +} + +.row-edit-bulk-preview[hidden] { + display: none; +} + +.row-edit-bulk-preview-summary { + color: var(--ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0; +} + +.row-edit-bulk-preview-table-wrap { + border: 1px solid var(--rule); + border-radius: 5px; + max-height: 18rem; + overflow: auto; + background: #fff; +} + +.row-edit-bulk-preview-table { + border-collapse: collapse; + font-size: 0.78rem; + min-width: 100%; + width: max-content; +} + +.row-edit-bulk-preview-table th, +.row-edit-bulk-preview-table td { + border-bottom: 1px solid var(--rule); + border-right: 1px solid var(--rule); + max-width: 18rem; + overflow-wrap: anywhere; + padding: 6px 8px; + text-align: left; + vertical-align: top; + white-space: normal; +} + +.row-edit-bulk-preview-table th { + background: var(--paper); + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} + +.row-edit-bulk-preview-table tr:last-child td { + border-bottom: none; +} + +.row-edit-bulk-preview-table th:last-child, +.row-edit-bulk-preview-table td:last-child { + border-right: none; +} + +.row-edit-bulk-preview-null, +.row-edit-bulk-preview-auto { + color: var(--muted); + font-style: italic; +} + +.row-edit-bulk-progress { + display: grid; + gap: 6px; +} + +.row-edit-bulk-progress[hidden] { + display: none; +} + +.row-edit-bulk-progress-bar { + width: 100%; +} + +.row-edit-bulk-progress-status { + color: var(--ink); + font-size: 0.9rem; + margin: 0; +} + datasette-autocomplete { display: block; position: relative; @@ -1865,95 +2562,24 @@ datasette-autocomplete input[type="text"], max-width: 46rem; } -.row-edit-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); +.row-edit-mode-link { + color: var(--accent); + font-size: 0.9rem; + margin-right: auto; } -.row-edit-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; +.row-edit-mode-link[hidden] { + display: none; } -.row-edit-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.row-edit-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.row-edit-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.row-edit-dialog .btn-primary:hover { - background: #1949b8; -} - -.row-edit-dialog .btn:disabled { +.row-edit-dialog .modal-btn:disabled { opacity: 0.55; cursor: not-allowed; } dialog.table-create-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(980px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.table-create-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.table-create-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.table-create-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .table-create-dialog .modal-title { @@ -1961,9 +2587,6 @@ dialog.table-create-dialog::backdrop { align-items: center; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .table-create-form { @@ -1991,8 +2614,6 @@ dialog.table-create-dialog::backdrop { .table-create-fields { display: grid; gap: 18px; - padding: 16px 24px 24px; - overflow-y: auto; } .table-create-field { @@ -2072,6 +2693,120 @@ select.table-create-input { gap: 10px; } +.table-create-columns[hidden], +.table-create-data[hidden], +.table-create-data-editor[hidden], +.table-create-data-preview[hidden] { + display: none; +} + +.table-create-data, +.table-create-data-editor { + display: grid; + gap: 8px; +} + +.table-create-data-label { + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; +} + +.table-create-data-textarea { + min-height: 16rem; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.45; +} + +.table-create-data-textarea.table-create-data-drop-target { + border-color: var(--accent); + background: #f8fbff; + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.table-create-data-note { + color: var(--muted); + font-size: 0.82rem; + margin: 0; +} + +.table-create-data-note label, +.table-create-data-note .button-as-link { + font: inherit; +} + +.table-create-data-preview { + display: grid; + gap: 10px; +} + +.table-create-data-preview-summary { + color: var(--ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0; +} + +.table-create-data-pk-field { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 12px; + align-items: center; +} + +.table-create-data-preview-table-wrap { + border: 1px solid var(--rule); + border-radius: 5px; + max-height: 18rem; + overflow: auto; + background: #fff; +} + +.table-create-data-preview-table { + border-collapse: collapse; + font-size: 0.78rem; + min-width: 100%; + width: max-content; +} + +.table-create-data-preview-table th, +.table-create-data-preview-table td { + border-bottom: 1px solid var(--rule); + border-right: 1px solid var(--rule); + max-width: 18rem; + overflow-wrap: anywhere; + padding: 6px 8px; + text-align: left; + vertical-align: top; + white-space: normal; +} + +.table-create-data-preview-table th { + background: var(--paper); + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} + +.table-create-data-preview-table tr:last-child td { + border-bottom: none; +} + +.table-create-data-preview-table th:last-child, +.table-create-data-preview-table td:last-child { + border-right: none; +} + +.table-create-data-preview-null { + color: var(--muted); + font-style: italic; +} + .table-create-column-list { display: grid; gap: 8px; @@ -2288,50 +3023,17 @@ select.table-create-input { outline-offset: 1px; } -.table-create-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); +.table-create-mode-link { + color: var(--accent); + font-size: 0.9rem; + margin-right: auto; } -.table-create-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; +.table-create-mode-link[hidden] { + display: none; } -.table-create-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.table-create-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.table-create-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.table-create-dialog .btn-primary:hover { - background: #1949b8; -} - -.table-create-dialog .btn:disabled, +.table-create-dialog .modal-btn:disabled, .table-create-add-column:disabled, .table-create-icon-button:disabled { opacity: 0.55; @@ -2339,46 +3041,8 @@ select.table-create-input { } dialog.table-alter-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(980px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.table-alter-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.table-alter-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.table-alter-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .table-alter-dialog .modal-title { @@ -2386,9 +3050,6 @@ dialog.table-alter-dialog::backdrop { align-items: center; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .table-alter-form { @@ -2416,8 +3077,6 @@ dialog.table-alter-dialog::backdrop { .table-alter-fields { display: grid; gap: 18px; - padding: 16px 24px 24px; - overflow-y: auto; } .table-alter-table-options { @@ -2451,8 +3110,6 @@ dialog.table-alter-dialog::backdrop { .table-alter-review { display: grid; gap: 12px; - overflow-y: auto; - padding: 16px 24px 24px; } .table-alter-review[hidden] { @@ -2746,72 +3403,29 @@ select.table-alter-input { outline-offset: 1px; } -.table-alter-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.table-alter-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.table-alter-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.table-alter-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.table-alter-dialog .btn-danger { +.table-alter-dialog .modal-btn-danger { background: #b91c1c; color: #fff; margin-right: auto; } -.table-alter-dialog .btn-danger:hover { +.table-alter-dialog .modal-btn-danger:hover { background: #991b1b; } -.table-alter-dialog .btn-danger:disabled, -.table-alter-dialog .btn-danger:disabled:hover { +.table-alter-dialog .modal-btn-danger:disabled, +.table-alter-dialog .modal-btn-danger:disabled:hover { background: #d98c8c; color: #fff; } -.table-alter-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.table-alter-dialog .btn-primary:hover { - background: #1949b8; -} - -.table-alter-dialog .btn-primary:disabled, -.table-alter-dialog .btn-primary:disabled:hover { +.table-alter-dialog .modal-btn-primary:disabled, +.table-alter-dialog .modal-btn-primary:disabled:hover { background: #a0aec0; color: #fff; } -.table-alter-dialog .btn:disabled, +.table-alter-dialog .modal-btn:disabled, .table-alter-add-column:disabled, .table-alter-icon-button:disabled { opacity: 0.55; @@ -3012,7 +3626,8 @@ select.table-alter-input { .row-edit-dialog .modal-header, .row-edit-summary, .row-edit-loading, - .row-edit-fields { + .row-edit-fields, + .row-edit-bulk { padding-left: 18px; padding-right: 18px; } @@ -3031,6 +3646,15 @@ select.table-alter-input { padding-top: 0; } + .row-edit-bulk-conflict { + grid-template-columns: 1fr; + gap: 5px; + } + + .row-edit-bulk-conflict-label { + padding-top: 0; + } + .row-edit-dialog .modal-footer { padding-left: 18px; padding-right: 18px; @@ -3062,6 +3686,11 @@ select.table-alter-input { padding-top: 0; } + .table-create-data-pk-field { + grid-template-columns: 1fr; + gap: 5px; + } + .table-create-column-headings { display: none; } diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index 198641f3..29729f27 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -1,7 +1,9 @@ +let columnChooserInstanceCounter = 0; + class ColumnChooser extends HTMLElement { constructor() { super(); - this.attachShadow({ mode: "open" }); + this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`; // State this._items = []; @@ -26,375 +28,60 @@ class ColumnChooser extends HTMLElement { // Bound handlers this._onMove = this._onMove.bind(this); this._onUp = this._onUp.bind(this); + } - this.shadowRoot.innerHTML = ` - - - + connectedCallback() { + if (this._modal) return; + this.innerHTML = ` +
- Choose columns - + Choose columns +
- - + +
-
-
-
-
    +
    +
    +
    +
      - - - + + +
      -
      +       `; // DOM refs - this._dialog = this.shadowRoot.querySelector("dialog"); - this._listWrap = this.shadowRoot.getElementById("listWrap"); - this._dragList = this.shadowRoot.getElementById("dragList"); - this._pulseTop = this.shadowRoot.getElementById("pulseTop"); - this._pulseBot = this.shadowRoot.getElementById("pulseBot"); - this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); - this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); - this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); - this._applyBtn = this.shadowRoot.getElementById("applyBtn"); - this._countEl = this.shadowRoot.getElementById("selectedCount"); - this._footerEl = this.shadowRoot.getElementById("footerInfo"); + this._modal = this.querySelector("datasette-modal"); + this._listWrap = this.querySelector(".list-wrap"); + this._dragList = this.querySelector(".drag-list"); + this._pulseTop = this.querySelector(".scroll-pulse.top"); + this._pulseBot = this.querySelector(".scroll-pulse.bot"); + this._selectAllBtn = this.querySelector(".select-all"); + this._deselectAllBtn = this.querySelector(".deselect-all"); + this._cancelBtn = this.querySelector(".modal-btn-ghost"); + this._applyBtn = this.querySelector(".modal-btn-primary"); + this._countEl = this.querySelector(".modal-meta"); + this._footerEl = this.querySelector(".footer-info"); // Event listeners this._selectAllBtn.addEventListener("click", () => this._selectAll()); this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); - this._cancelBtn.addEventListener("click", () => this._close()); + this._cancelBtn.addEventListener("click", () => + this._modal.requestClose("cancel"), + ); this._applyBtn.addEventListener("click", () => this._apply()); - this._dialog.addEventListener("click", (e) => { - if (e.target === this._dialog) this._close(); - }); - this._dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this._close(); - }); + this._modal.beforeClose = () => { + this._items = this._savedItems ? [...this._savedItems] : this._items; + this._checked = this._savedChecked + ? new Set(this._savedChecked) + : this._checked; + return true; + }; } /** @@ -414,19 +101,11 @@ class ColumnChooser extends HTMLElement { this._savedChecked = new Set(this._checked); this._render(); - this._dialog.showModal(); + this._modal.show(); } // ── Internal methods ── - _close() { - this._items = this._savedItems ? [...this._savedItems] : this._items; - this._checked = this._savedChecked - ? new Set(this._savedChecked) - : this._checked; - this._dialog.close(); - } - _selectAll() { this._items.forEach((col) => this._checked.add(col)); this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { @@ -445,7 +124,7 @@ class ColumnChooser extends HTMLElement { _apply() { const selected = this._items.filter((col) => this._checked.has(col)); - this._dialog.close(); + this._modal.close(); if (this._onApply) { this._onApply(selected); } @@ -472,11 +151,13 @@ class ColumnChooser extends HTMLElement { - ${col} +
      `; + li.querySelector(".drag-item-label").textContent = col; + li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); @@ -509,7 +190,7 @@ class ColumnChooser extends HTMLElement { this._ghostOffX = e.clientX - rect.left; this._ghostOffY = e.clientY - rect.top; - // Build ghost inside shadow DOM + // Keep the drag preview inside the dialog so it stays above the backdrop. this._ghost = document.createElement("div"); this._ghost.className = "drag-ghost"; this._ghost.style.width = rect.width + "px"; @@ -518,7 +199,7 @@ class ColumnChooser extends HTMLElement { this._ghost.querySelector(".drop-indicator")?.remove(); const h = this._ghost.querySelector(".drag-handle"); if (h) h.style.color = "var(--accent)"; - this.shadowRoot.appendChild(this._ghost); + this._modal.dialog.appendChild(this._ghost); srcEl.classList.add("is-dragging"); this._positionGhost(e.clientX, e.clientY); diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 9f4f89b9..0f61ebd6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -2,8 +2,10 @@ var ROW_DELETE_DIALOG_ID = "row-delete-dialog"; var rowDeleteDialogState = null; var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; +var ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES = 10 * 1024 * 1024; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; +var TABLE_CREATE_AUTOMATIC_PK = "__datasette_automatic_pk__"; var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; var tableAlterDialogState = null; @@ -810,12 +812,60 @@ async function loadTableCreateForeignKeyTargets(state) { ); } +function tableCreateIsDataMode(state) { + return state && state.mode === "data"; +} + +function tableCreateSaveButtonText(state) { + if (tableCreateIsDataMode(state)) { + return state.dataPreviewReady ? "Create table" : "Preview rows"; + } + return "Create table"; +} + +function tableCreateCanInsertRows() { + var data = databaseCreateTableData() || {}; + return !!data.canInsertRows; +} + +function syncTableCreateModeUi(state) { + if (!state) { + return; + } + var isDataMode = tableCreateIsDataMode(state); + state.columnsPanel.hidden = isDataMode; + state.dataPanel.hidden = !isDataMode; + state.dataEditor.hidden = !isDataMode || state.dataPreviewReady; + state.dataPreview.hidden = !isDataMode || !state.dataPreviewReady; + state.createFromDataLink.hidden = isDataMode || !tableCreateCanInsertRows(); + state.manualCreateLink.hidden = !isDataMode; +} + +function updateTableCreateDialogButtons(state) { + if (!state) { + return; + } + syncTableCreateModeUi(state); + state.cancelButton.disabled = state.isSaving; + state.saveButton.disabled = state.isSaving; + state.addColumnButton.disabled = state.isSaving; + state.cancelButton.textContent = + tableCreateIsDataMode(state) && state.dataPreviewReady ? "Back" : "Cancel"; + state.saveButton.textContent = state.isSaving + ? "Creating..." + : tableCreateSaveButtonText(state); +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; } - return JSON.stringify({ + var signature = { table: state.tableName.value, + data: state.dataTextarea ? state.dataTextarea.value : "", + dataPrimaryKey: state.dataPkSelect + ? state.dataPkSelect.value + : TABLE_CREATE_AUTOMATIC_PK, columns: tableCreateDialogRows(state).map(function (row) { return { name: row.querySelector(".table-create-column-name").value, @@ -838,7 +888,8 @@ function tableCreateDialogSignature(state) { ).value || "", }; }), - }); + }; + return JSON.stringify(signature); } function tableCreateDialogHasChanges(state) { @@ -864,18 +915,24 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.cancelButton.disabled = isSaving; - state.saveButton.disabled = isSaving; - state.addColumnButton.disabled = isSaving; - state.saveButton.textContent = isSaving ? "Creating..." : "Create table"; + state.modal.busy = isSaving; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { control.disabled = isSaving; }); + state.fields + .querySelectorAll( + ".table-create-data input, .table-create-data select, .table-create-data textarea, .table-create-data button", + ) + .forEach(function (control) { + control.disabled = isSaving; + }); + state.tableName.disabled = isSaving; if (!isSaving) { updateTableCreateColumnRules(state); } + updateTableCreateDialogButtons(state); updateTableCreateMoveButtons(state); } @@ -1231,8 +1288,11 @@ function addTableCreateColumn(state, column) { } function resetTableCreateDialog(state) { + state.mode = "manual"; state.nextColumnIndex = 0; state.tableName.value = ""; + state.dataTextarea.value = ""; + resetTableCreateDataPreview(state); state.columnList.textContent = ""; addTableCreateColumn(state, { name: "id", @@ -1245,9 +1305,39 @@ function resetTableCreateDialog(state) { primaryKey: false, }); updateTableCreateColumnRules(state); + updateTableCreateDialogButtons(state); state.initialSignature = tableCreateDialogSignature(state); } +function showTableCreateDataMode(state) { + if (!state || state.isSaving || !tableCreateCanInsertRows()) { + return; + } + state.mode = "data"; + clearTableCreateDialogError(state); + updateTableCreateDialogButtons(state); + if (state.dataPreviewReady && state.dataPkSelect) { + state.dataPkSelect.focus(); + } else { + state.dataTextarea.focus(); + } +} + +function showTableCreateManualMode(state) { + if (!state || state.isSaving) { + return; + } + state.mode = "manual"; + clearTableCreateDialogError(state); + updateTableCreateDialogButtons(state); + var firstInput = state.columnList.querySelector(".table-create-column-name"); + if (firstInput) { + firstInput.focus(); + } else { + state.tableName.focus(); + } +} + function collectTableCreatePayload(state) { var payload = { table: state.tableName.value.trim(), @@ -1315,14 +1405,9 @@ function collectTableCreateColumnTypeAssignments(state) { } function validateTableCreatePayload(payload) { - if (!payload.table) { - return "Table name is required."; - } - if (payload.table.indexOf("\n") !== -1) { - return "Table name cannot contain newlines."; - } - if (/^sqlite_/i.test(payload.table)) { - return "Table name cannot start with sqlite_."; + var tableNameError = validateTableCreateTableName(payload.table); + if (tableNameError) { + return tableNameError; } if (!payload.columns.length) { return "At least one column is required."; @@ -1352,6 +1437,19 @@ function validateTableCreatePayload(payload) { return null; } +function validateTableCreateTableName(tableName) { + if (!tableName) { + return "Table name is required."; + } + if (tableName.indexOf("\n") !== -1) { + return "Table name cannot contain newlines."; + } + if (/^sqlite_/i.test(tableName)) { + return "Table name cannot start with sqlite_."; + } + return null; +} + function validateTableCreateColumnTypeAssignments(assignments) { for (var i = 0; i < assignments.length; i += 1) { var assignment = assignments[i]; @@ -1374,6 +1472,476 @@ function validateTableCreateColumnTypeAssignments(assignments) { return null; } +function normalizeCreateTableDataJsonValue(value) { + if (typeof value === "undefined") { + return null; + } + if (Array.isArray(value) || (value && typeof value === "object")) { + return JSON.stringify(value); + } + return value; +} + +function createTableDataColumnObjects(names) { + return names.map(function (name) { + return { name: name }; + }); +} + +function validateCreateTableDataHeaders(headers) { + if (!headers.length) { + throw new Error("No columns found to preview."); + } + var seen = {}; + headers.forEach(function (name, index) { + if (!name) { + throw new Error("Column header " + (index + 1) + " is blank."); + } + if (name.indexOf("\n") !== -1) { + throw new Error("Column names cannot contain newlines."); + } + var key = name.toLowerCase(); + if (seen[key]) { + throw new Error("Duplicate column name: " + name); + } + seen[key] = true; + }); +} + +function jsonRowIsObject(item) { + return !!(item && typeof item === "object" && !Array.isArray(item)); +} + +function extractJsonObjectRows(parsed) { + if (Array.isArray(parsed)) { + return parsed; + } + if (!jsonRowIsObject(parsed)) { + throw new Error( + "JSON must be an array of objects, or an object containing an array of objects.", + ); + } + + var bestRows = null; + Object.keys(parsed).forEach(function (key) { + var value = parsed[key]; + if (!Array.isArray(value) || !value.every(jsonRowIsObject)) { + return; + } + if (!bestRows || value.length > bestRows.length) { + bestRows = value; + } + }); + if (!bestRows) { + throw new Error( + "JSON object must contain at least one root key with an array of objects.", + ); + } + return bestRows; +} + +function parseJsonObjectRows(text) { + var parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new Error("Invalid JSON: " + error.message); + } + var rows = extractJsonObjectRows(parsed); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + return rows; +} + +function parseJsonCreateTableRows(text) { + var parsed = parseJsonObjectRows(text); + + var columnNames = []; + var columnMap = {}; + parsed.forEach(function (item, index) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("JSON row " + (index + 1) + " must be an object."); + } + Object.keys(item).forEach(function (name) { + if (!columnMap[name]) { + columnMap[name] = true; + columnNames.push(name); + } + }); + }); + validateCreateTableDataHeaders(columnNames); + + var rows = parsed.map(function (item) { + var row = {}; + columnNames.forEach(function (name) { + row[name] = Object.prototype.hasOwnProperty.call(item, name) + ? normalizeCreateTableDataJsonValue(item[name]) + : null; + }); + return row; + }); + return { + columns: createTableDataColumnObjects(columnNames), + rows: rows, + }; +} + +function createTableDelimitedValueIsInteger(value) { + return /^[-+]?\d+$/.test(String(value).trim()); +} + +function createTableDelimitedValueIsFloat(value) { + var trimmed = String(value).trim(); + if (!trimmed) { + return false; + } + var numberValue = Number(trimmed); + return Number.isFinite(numberValue); +} + +function inferCreateTableDelimitedColumnType(values) { + var nonBlankValues = values.filter(function (value) { + return String(value).trim() !== ""; + }); + if (!nonBlankValues.length) { + return "text"; + } + if (nonBlankValues.every(createTableDelimitedValueIsInteger)) { + return "integer"; + } + if (nonBlankValues.every(createTableDelimitedValueIsFloat)) { + return "float"; + } + return "text"; +} + +function coerceCreateTableDelimitedValue(value, type) { + var trimmed = String(value).trim(); + if (trimmed === "") { + return type === "integer" || type === "float" ? null : ""; + } + if (type === "integer") { + return parseInt(trimmed, 10); + } + if (type === "float") { + return Number(trimmed); + } + return value; +} + +function detectCreateTableDataDelimiter(text) { + var firstLine = + text.split(/\r\n|\n|\r/).find(function (line) { + return line.trim() !== ""; + }) || ""; + var csvRows = delimiterPreviewRows(firstLine, ","); + var tsvRows = delimiterPreviewRows(firstLine, "\t"); + var csvColumns = csvRows.length ? csvRows[0].length : 0; + var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; + + if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { + return "\t"; + } + if (tsvColumns > csvColumns) { + return "\t"; + } + if (csvColumns > 1) { + return ","; + } + if (tsvColumns > 1) { + return "\t"; + } + return null; +} + +function parseDelimitedCreateTableRows(text) { + var delimiter = detectCreateTableDataDelimiter(text); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + + var headers = rows[0].map(function (value) { + return value.trim(); + }); + validateCreateTableDataHeaders(headers); + var dataRows = rows.slice(1); + if (!dataRows.length) { + throw new Error("No data rows found to preview."); + } + + dataRows.forEach(function (row, index) { + if (row.length > headers.length) { + throw new Error( + "Row " + + (index + 1) + + " has " + + row.length + + " values, but only " + + headers.length + + " columns were provided.", + ); + } + }); + + var columnTypes = headers.map(function (_name, columnIndex) { + return inferCreateTableDelimitedColumnType( + dataRows.map(function (row) { + return row[columnIndex] || ""; + }), + ); + }); + + return { + columns: createTableDataColumnObjects(headers), + rows: dataRows.map(function (row) { + var rowObject = {}; + headers.forEach(function (name, columnIndex) { + rowObject[name] = coerceCreateTableDelimitedValue( + row[columnIndex] || "", + columnTypes[columnIndex], + ); + }); + return rowObject; + }), + }; +} + +function parseCreateTableDataRows(text) { + var trimmed = text.trim(); + if (!trimmed) { + throw new Error("Paste rows before previewing."); + } + if (trimmed[0] === "[" || trimmed[0] === "{") { + return parseJsonCreateTableRows(trimmed); + } + return parseDelimitedCreateTableRows(trimmed); +} + +function tableCreateDataRecommendedPrimaryKey(columns, rows) { + var candidates = []; + columns.forEach(function (column, columnIndex) { + var distinctValues = {}; + var maxLength = 0; + var valid = rows.length > 0; + rows.forEach(function (row) { + if (!valid) { + return; + } + var value = row[column.name]; + if (value === null || typeof value === "undefined") { + valid = false; + return; + } + var text = String(value).trim(); + if (!text || text.length >= 20 || /\s/.test(text)) { + valid = false; + return; + } + if (distinctValues[text]) { + valid = false; + return; + } + distinctValues[text] = true; + maxLength = Math.max(maxLength, text.length); + }); + if (valid) { + candidates.push({ + name: column.name, + maxLength: maxLength, + columnIndex: columnIndex, + }); + } + }); + candidates.sort(function (left, right) { + if (left.maxLength !== right.maxLength) { + return left.maxLength - right.maxLength; + } + return left.columnIndex - right.columnIndex; + }); + return candidates.length ? candidates[0].name : TABLE_CREATE_AUTOMATIC_PK; +} + +function renderTableCreateDataPreview(state, preview) { + state.dataPreview.textContent = ""; + + var summary = document.createElement("p"); + summary.className = "table-create-data-preview-summary"; + summary.textContent = + "Previewing " + + preview.rows.length + + " row" + + (preview.rows.length === 1 ? "." : "s."); + state.dataPreview.appendChild(summary); + + var pkField = document.createElement("div"); + pkField.className = "table-create-data-pk-field"; + var pkLabel = document.createElement("label"); + pkLabel.className = "table-create-data-label"; + pkLabel.setAttribute("for", "table-create-data-primary-key"); + pkLabel.textContent = "Primary key"; + var pkSelect = document.createElement("select"); + pkSelect.id = "table-create-data-primary-key"; + pkSelect.className = "table-create-input table-create-data-primary-key"; + var automaticOption = document.createElement("option"); + automaticOption.value = TABLE_CREATE_AUTOMATIC_PK; + automaticOption.textContent = "Automatic ID column"; + pkSelect.appendChild(automaticOption); + preview.columns.forEach(function (column) { + var option = document.createElement("option"); + option.value = column.name; + option.textContent = column.name; + pkSelect.appendChild(option); + }); + pkSelect.value = tableCreateDataRecommendedPrimaryKey( + preview.columns, + preview.rows, + ); + pkSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + }); + state.dataPkSelect = pkSelect; + pkField.appendChild(pkLabel); + pkField.appendChild(pkSelect); + state.dataPreview.appendChild(pkField); + + var tableWrap = document.createElement("div"); + tableWrap.className = "table-create-data-preview-table-wrap"; + var table = document.createElement("table"); + table.className = "table-create-data-preview-table"; + var thead = document.createElement("thead"); + var headerRow = document.createElement("tr"); + preview.columns.forEach(function (column) { + var th = document.createElement("th"); + th.scope = "col"; + th.textContent = column.name; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + var tbody = document.createElement("tbody"); + preview.rows.forEach(function (row) { + var tr = document.createElement("tr"); + preview.columns.forEach(function (column) { + var td = document.createElement("td"); + var value = row[column.name]; + td.textContent = bulkInsertPreviewValue(value); + if (value === null) { + td.className = "table-create-data-preview-null"; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + tableWrap.appendChild(table); + state.dataPreview.appendChild(tableWrap); + state.dataPreview.hidden = false; +} + +function resetTableCreateDataPreview(state) { + state.dataPreviewRows = null; + state.dataPreviewColumns = []; + state.dataPreviewReady = false; + state.dataPkSelect = null; + state.dataPreview.hidden = true; + state.dataPreview.textContent = ""; + syncTableCreateModeUi(state); +} + +function tableCreateTableNameFromFileName(fileName) { + var baseName = (fileName || "").replace(/^.*[\\/]/, ""); + var nameWithoutExtension = baseName.replace(/\.[^.]*$/, ""); + return nameWithoutExtension + .trim() + .replace(/\s+/g, "_") + .toLowerCase() + .replace(/[^a-z0-9_]/g, ""); +} + +async function loadTableCreateDataTextFile(state, file) { + if (!file) { + return; + } + try { + var text = await readTextFile(file); + var tableName = tableCreateTableNameFromFileName(file.name); + if (tableName) { + state.tableName.value = tableName; + state.tableName.dispatchEvent(new Event("input", { bubbles: true })); + } + state.dataTextarea.value = text; + state.dataTextarea.dispatchEvent(new Event("input", { bubbles: true })); + clearTableCreateDialogError(state); + state.dataTextarea.focus(); + } catch (_error) { + showTableCreateDialogError(state, "Could not read that text file."); + } +} + +function previewTableCreateDataRows(state) { + clearTableCreateDialogError(state); + resetTableCreateDataPreview(state); + try { + var preview = parseCreateTableDataRows(state.dataTextarea.value); + state.dataPreviewRows = preview.rows; + state.dataPreviewColumns = preview.columns; + state.dataPreviewReady = true; + renderTableCreateDataPreview(state, preview); + updateTableCreateDialogButtons(state); + } catch (error) { + showTableCreateDialogError( + state, + error.message || "Could not preview rows.", + ); + updateTableCreateDialogButtons(state); + } +} + +function collectTableCreateDataPayload(state) { + var payload = { + table: state.tableName.value.trim(), + rows: state.dataPreviewRows || [], + }; + var primaryKey = state.dataPkSelect + ? state.dataPkSelect.value + : TABLE_CREATE_AUTOMATIC_PK; + payload.pk = + primaryKey === TABLE_CREATE_AUTOMATIC_PK ? "id" : primaryKey || undefined; + return payload; +} + +function validateTableCreateDataPayload(payload, state) { + var tableNameError = validateTableCreateTableName(payload.table); + if (tableNameError) { + return tableNameError; + } + if (!payload.rows.length) { + return "No rows found to create."; + } + if ( + state.dataPkSelect && + state.dataPkSelect.value === TABLE_CREATE_AUTOMATIC_PK && + payload.rows.some(function (row) { + return Object.prototype.hasOwnProperty.call(row, "id"); + }) + ) { + return ( + "Automatic ID column cannot be used because the pasted data " + + "already has an id column." + ); + } + return null; +} + function fallbackTableUrl(tableName) { var data = databaseCreateTableData() || {}; if (!data.path) { @@ -1441,6 +2009,56 @@ async function assignTableCreateColumnTypes( } } +async function createTableFromDataPreview(state) { + var data = databaseCreateTableData(); + if (!data || !data.path) { + showTableCreateDialogError(state, "Could not find the create table URL."); + return; + } + var payload = collectTableCreateDataPayload(state); + var validationError = validateTableCreateDataPayload(payload, state); + if (validationError) { + showTableCreateDialogError(state, validationError); + return; + } + clearTableCreateDialogError(state); + setTableCreateDialogSaving(state, true); + try { + var response = await fetch(data.path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + var tableUrl = + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); + state.modal.close({ restoreFocus: false }); + if (tableUrl) { + location.href = tableUrl; + } else { + location.reload(); + } + } catch (error) { + setTableCreateDialogSaving(state, false); + showTableCreateDialogError( + state, + error.message || "Could not create table", + ); + } +} + async function saveTableCreateDialog(state) { if (state.isSaving) { return; @@ -1450,6 +2068,14 @@ async function saveTableCreateDialog(state) { showTableCreateDialogError(state, "Could not find the create table URL."); return; } + if (tableCreateIsDataMode(state)) { + if (!state.dataPreviewReady) { + previewTableCreateDataRows(state); + } else { + await createTableFromDataPreview(state); + } + return; + } clearTableCreateDialogError(state); var payload = collectTableCreatePayload(state); var columnTypeAssignments = collectTableCreateColumnTypeAssignments(state); @@ -1492,8 +2118,7 @@ async function saveTableCreateDialog(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (tableUrl) { location.href = tableUrl; } else { @@ -1515,18 +2140,6 @@ function confirmDiscardTableCreateChanges(state) { return window.confirm("Discard this new table?"); } -function closeTableCreateDialogIfConfirmed(state) { - if (!state || state.isSaving) { - return false; - } - if (!confirmDiscardTableCreateChanges(state)) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - function ensureTableCreateDialog(manager) { if (tableCreateDialogState) { return tableCreateDialogState; @@ -1535,7 +2148,8 @@ function ensureTableCreateDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = TABLE_CREATE_DIALOG_ID; dialog.className = "table-create-dialog"; dialog.setAttribute("aria-labelledby", "table-create-title"); @@ -1545,7 +2159,7 @@ function ensureTableCreateDialog(manager) {
      -
      +
      @@ -1560,29 +2174,52 @@ function ensureTableCreateDialog(manager) {
      +
      - - + Create table from data + + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); tableCreateDialogState = { + modal: modal, dialog: dialog, form: dialog.querySelector(".table-create-form"), title: dialog.querySelector(".modal-title"), error: dialog.querySelector(".table-create-error"), fields: dialog.querySelector(".table-create-fields"), tableName: dialog.querySelector(".table-create-table-name"), + columnsPanel: dialog.querySelector(".table-create-columns"), columnList: dialog.querySelector(".table-create-column-list"), addColumnButton: dialog.querySelector(".table-create-add-column"), + dataPanel: dialog.querySelector(".table-create-data"), + dataEditor: dialog.querySelector(".table-create-data-editor"), + dataTextarea: dialog.querySelector(".table-create-data-textarea"), + dataOpenFileButton: dialog.querySelector(".table-create-data-open-file"), + dataFileInput: dialog.querySelector(".table-create-data-file-input"), + dataPreview: dialog.querySelector(".table-create-data-preview"), + createFromDataLink: dialog.querySelector(".table-create-from-data"), + manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), - currentButton: null, - shouldRestoreFocus: true, isSaving: false, + mode: "manual", + dataPreviewRows: null, + dataPreviewColumns: [], + dataPreviewReady: false, + dataPkSelect: null, initialSignature: "", nextColumnIndex: 0, foreignKeyTargets: [], @@ -1606,43 +2243,122 @@ function ensureTableCreateDialog(manager) { }); tableCreateDialogState.cancelButton.addEventListener("click", function () { - closeTableCreateDialogIfConfirmed(tableCreateDialogState); + if ( + tableCreateIsDataMode(tableCreateDialogState) && + tableCreateDialogState.dataPreviewReady && + !tableCreateDialogState.isSaving + ) { + resetTableCreateDataPreview(tableCreateDialogState); + updateTableCreateDialogButtons(tableCreateDialogState); + tableCreateDialogState.dataTextarea.focus(); + return; + } + modal.requestClose("cancel"); }); + tableCreateDialogState.createFromDataLink.addEventListener( + "click", + function (ev) { + ev.preventDefault(); + showTableCreateDataMode(tableCreateDialogState); + }, + ); + + tableCreateDialogState.manualCreateLink.addEventListener( + "click", + function (ev) { + ev.preventDefault(); + showTableCreateManualMode(tableCreateDialogState); + }, + ); + tableCreateDialogState.tableName.addEventListener("input", function () { clearTableCreateDialogError(tableCreateDialogState); }); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeTableCreateDialogIfConfirmed(tableCreateDialogState); - } + tableCreateDialogState.dataOpenFileButton.addEventListener( + "click", + function () { + tableCreateDialogState.dataFileInput.click(); + }, + ); + + tableCreateDialogState.dataFileInput.addEventListener( + "change", + async function (ev) { + var files = ev.target.files; + await loadTableCreateDataTextFile( + tableCreateDialogState, + files && files.length ? files[0] : null, + ); + ev.target.value = ""; + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragenter", + function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.add( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragover", + function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.add( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragleave", + function () { + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "drop", + async function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + var files = ev.dataTransfer && ev.dataTransfer.files; + if (!files || !files.length) { + return; + } + await loadTableCreateDataTextFile(tableCreateDialogState, files[0]); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener("dragend", function () { + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); }); - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { - return; - } - ev.preventDefault(); - closeTableCreateDialogIfConfirmed(tableCreateDialogState); + tableCreateDialogState.dataTextarea.addEventListener("input", function () { + resetTableCreateDataPreview(tableCreateDialogState); + clearTableCreateDialogError(tableCreateDialogState); + updateTableCreateDialogButtons(tableCreateDialogState); }); - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeTableCreateDialogIfConfirmed(tableCreateDialogState); - }); + modal.beforeClose = function (source) { + return confirmDiscardTableCreateChanges(tableCreateDialogState); + }; dialog.addEventListener("close", function () { var state = tableCreateDialogState; clearTableCreateDialogError(state); setTableCreateDialogSaving(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } }); return tableCreateDialogState; @@ -1663,15 +2379,12 @@ function openTableCreateDialog(button, manager) { menu.open = false; } state.manager = manager; - state.currentButton = button; - state.shouldRestoreFocus = true; + state.title.textContent = "Create a table in " + data.databaseName; clearTableCreateDialogError(state); resetTableCreateDialog(state); loadTableCreateForeignKeyTargets(state); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); state.tableName.focus(); } @@ -1697,6 +2410,7 @@ function initTableCreateActions(manager) { function setRowDeleteDialogBusy(state, isBusy) { state.isBusy = isBusy; + state.modal.busy = isBusy; state.confirmButton.disabled = isBusy; state.cancelButton.disabled = isBusy; state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row"; @@ -1943,6 +2657,7 @@ function showTableAlterDialogError(state, message) { function setTableAlterDialogSaving(state, isSaving) { state.isSaving = isSaving; + state.modal.busy = isSaving; state.cancelButton.disabled = isSaving; state.addColumnButton.disabled = isSaving; state.backButton.disabled = isSaving; @@ -3078,8 +3793,7 @@ async function applyTableAlterChanges(state, result) { result.columnTypeAssignments || [], tableUrl, ); - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (tableAlterResultRenamesTable(result) && tableUrl) { window.location.href = tableUrl; } else { @@ -3140,8 +3854,7 @@ async function dropTableFromAlterDialog(state) { if (!response.ok || (responseData && responseData.ok === false)) { throw rowMutationRequestError(response, responseData); } - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); window.location.href = tableAlterDatabaseUrl() || "/"; } catch (error) { setTableAlterDialogSaving(state, false); @@ -3177,27 +3890,6 @@ function confirmDiscardTableAlterChanges(state) { return window.confirm("Discard table changes?"); } -function closeTableAlterDialogIfConfirmed(state) { - if (!state || state.isSaving) { - return false; - } - if (!confirmDiscardTableAlterChanges(state)) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - -function closeTableAlterDialog(state) { - if (!state || state.isSaving) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - function ensureTableAlterDialog(manager) { if (tableAlterDialogState) { return tableAlterDialogState; @@ -3206,7 +3898,8 @@ function ensureTableAlterDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = TABLE_ALTER_DIALOG_ID; dialog.className = "table-alter-dialog"; dialog.setAttribute("aria-labelledby", "table-alter-title"); @@ -3216,7 +3909,7 @@ function ensureTableAlterDialog(manager) {
      -
      +
      - +
      - - - - + + + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); tableAlterDialogState = { + modal: modal, dialog: dialog, form: dialog.querySelector(".table-alter-form"), title: dialog.querySelector(".modal-title"), @@ -3261,8 +3955,6 @@ function ensureTableAlterDialog(manager) { dropButton: dialog.querySelector(".table-alter-drop"), cancelButton: dialog.querySelector(".table-alter-cancel"), saveButton: dialog.querySelector(".table-alter-save"), - currentButton: null, - shouldRestoreFocus: true, isSaving: false, initialSignature: "", originalTableName: "", @@ -3304,7 +3996,7 @@ function ensureTableAlterDialog(manager) { }); tableAlterDialogState.cancelButton.addEventListener("click", function () { - closeTableAlterDialog(tableAlterDialogState); + modal.requestClose("cancel"); }); tableAlterDialogState.dropButton.addEventListener("click", function () { @@ -3325,36 +4017,17 @@ function ensureTableAlterDialog(manager) { } }); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeTableAlterDialogIfConfirmed(tableAlterDialogState); - } - }); - - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { - return; - } - ev.preventDefault(); - closeTableAlterDialogIfConfirmed(tableAlterDialogState); - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeTableAlterDialogIfConfirmed(tableAlterDialogState); - }); + modal.beforeClose = function (source) { + return ( + source === "cancel" || + confirmDiscardTableAlterChanges(tableAlterDialogState) + ); + }; dialog.addEventListener("close", function () { var state = tableAlterDialogState; clearTableAlterDialogError(state); setTableAlterDialogSaving(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } }); return tableAlterDialogState; @@ -3375,8 +4048,7 @@ function openTableAlterDialog(button, manager) { menu.open = false; } state.manager = manager; - state.currentButton = button; - state.shouldRestoreFocus = true; + state.title.textContent = "Alter table " + data.tableName; clearTableAlterDialogError(state); resetTableAlterDialog(state, data); @@ -3386,9 +4058,7 @@ function openTableAlterDialog(button, manager) { tableAlterForeignKeyTargetsUrl(), { filterByType: false }, ); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); var firstName = state.columnList.querySelector(".table-alter-column-name"); if (firstName) { firstName.focus(); @@ -3582,6 +4252,14 @@ function tableInsertUrl() { return url.toString(); } +function tableUpsertUrl() { + var data = tableInsertData(); + if (data && data.upsertPath) { + return new URL(data.upsertPath, location.href).toString(); + } + return null; +} + function rowResourceUrl(row) { var rowId = row.getAttribute("data-row"); if (!rowId) { @@ -3598,7 +4276,7 @@ function rowJsonUrl(row) { return ""; } url.pathname = url.pathname + ".json"; - url.searchParams.set("_extra", "columns,column_types"); + url.searchParams.set("_extra", "columns,column_types,column_details"); return url.toString(); } @@ -3683,7 +4361,8 @@ function ensureRowDeleteDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = ROW_DELETE_DIALOG_ID; dialog.className = "row-delete-dialog"; dialog.setAttribute("aria-labelledby", "row-delete-title"); @@ -3695,13 +4374,14 @@ function ensureRowDeleteDialog(manager) {

      Delete row ?

      - - + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); rowDeleteDialogState = { + modal: modal, dialog: dialog, title: dialog.querySelector(".modal-title"), message: dialog.querySelector(".row-delete-message"), @@ -3714,21 +4394,10 @@ function ensureRowDeleteDialog(manager) { currentPkPath: null, manager: manager, isBusy: false, - shouldRestoreFocus: true, }; rowDeleteDialogState.cancelButton.addEventListener("click", function () { - if (!rowDeleteDialogState.isBusy) { - rowDeleteDialogState.shouldRestoreFocus = true; - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !rowDeleteDialogState.isBusy) { - rowDeleteDialogState.shouldRestoreFocus = true; - dialog.close(); - } + modal.requestClose("cancel"); }); dialog.addEventListener("keydown", function (ev) { @@ -3740,25 +4409,6 @@ function ensureRowDeleteDialog(manager) { if (!rowDeleteDialogState.isBusy) { rowDeleteDialogState.confirmButton.click(); } - return; - } - if (ev.key !== "Escape") { - return; - } - if (rowDeleteDialogState.isBusy) { - ev.preventDefault(); - return; - } - ev.preventDefault(); - rowDeleteDialogState.shouldRestoreFocus = true; - dialog.close(); - }); - - dialog.addEventListener("cancel", function (ev) { - if (rowDeleteDialogState.isBusy) { - ev.preventDefault(); - } else { - rowDeleteDialogState.shouldRestoreFocus = true; } }); @@ -3766,13 +4416,6 @@ function ensureRowDeleteDialog(manager) { var state = rowDeleteDialogState; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } }); rowDeleteDialogState.confirmButton.addEventListener( @@ -3799,8 +4442,7 @@ function ensureRowDeleteDialog(manager) { throw rowMutationRequestError(response, data); } if (data && data.redirect) { - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); location.href = data.redirect; return; } @@ -3812,8 +4454,7 @@ function ensureRowDeleteDialog(manager) { var statusMessage = state.currentPkPath ? "Deleted row " + state.currentPkPath + "." : "Deleted row."; - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); state.currentRow.remove(); showRowMutationStatus(state.manager, statusMessage, false); if (focusTarget && document.contains(focusTarget)) { @@ -3842,11 +4483,9 @@ function openRowDeleteDialog(button, manager) { } state.manager = manager; - state.currentButton = button; state.currentRow = row; state.currentDeleteUrl = rowDeleteUrl(row); state.currentPkPath = rowDisplayLabel(row); - state.shouldRestoreFocus = true; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); @@ -3858,9 +4497,7 @@ function openRowDeleteDialog(button, manager) { ); state.rowId.textContent = state.currentPkPath || "this row"; - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); state.confirmButton.focus(); } @@ -3878,10 +4515,155 @@ function initRowDeleteActions(manager) { }); } +function isBase64JsonValue(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + var keys = Object.keys(value); + return ( + keys.length === 2 && + Object.prototype.hasOwnProperty.call(value, "$base64") && + Object.prototype.hasOwnProperty.call(value, "encoded") && + value.$base64 === true && + typeof value.encoded === "string" + ); +} + +function shouldUseBinaryControl(value, options) { + options = options || {}; + var sqliteType = (options.sqliteType || "").toLowerCase(); + return ( + isBase64JsonValue(value) || + sqliteType === "blob" || + options.valueKind === "binary" + ); +} + +function binaryEncodedValue(value) { + return isBase64JsonValue(value) ? value.encoded : ""; +} + +function binaryByteLengthFromBase64(encoded) { + encoded = (encoded || "").replace(/\s/g, ""); + if (!encoded) { + return 0; + } + var padding = 0; + if (encoded.slice(-2) === "==") { + padding = 2; + } else if (encoded.slice(-1) === "=") { + padding = 1; + } + return Math.max(0, Math.floor((encoded.length * 3) / 4) - padding); +} + +function rowEditBinaryValue(encoded) { + return { + $base64: true, + encoded: encoded || "", + }; +} + +function formatRowEditBinarySize(byteLength) { + var number = byteLength.toLocaleString(); + return "Binary: " + number + " byte" + (byteLength === 1 ? "" : "s"); +} + +function base64ToUint8Array(encoded) { + var binary = window.atob(encoded || ""); + var bytes = new Uint8Array(binary.length); + for (var i = 0; i < binary.length; i += 1) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +function uint8ArrayToBase64(bytes) { + var chunks = []; + var chunkSize = 0x8000; + for (var i = 0; i < bytes.length; i += chunkSize) { + chunks.push( + String.fromCharCode.apply(null, bytes.subarray(i, i + chunkSize)), + ); + } + return window.btoa(chunks.join("")); +} + +function rowEditBinaryImageMimeType(bytes) { + if (!bytes || !bytes.length) { + return null; + } + if ( + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 && + bytes[4] === 0x0d && + bytes[5] === 0x0a && + bytes[6] === 0x1a && + bytes[7] === 0x0a + ) { + return "image/png"; + } + if ( + bytes.length >= 3 && + bytes[0] === 0xff && + bytes[1] === 0xd8 && + bytes[2] === 0xff + ) { + return "image/jpeg"; + } + if ( + bytes.length >= 6 && + bytes[0] === 0x47 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x38 && + (bytes[4] === 0x37 || bytes[4] === 0x39) && + bytes[5] === 0x61 + ) { + return "image/gif"; + } + if ( + bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ) { + return "image/webp"; + } + if ( + bytes.length >= 12 && + bytes[4] === 0x66 && + bytes[5] === 0x74 && + bytes[6] === 0x79 && + bytes[7] === 0x70 && + bytes[8] === 0x61 && + bytes[9] === 0x76 && + bytes[10] === 0x69 && + (bytes[11] === 0x66 || bytes[11] === 0x73) + ) { + return "image/avif"; + } + if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) { + return "image/bmp"; + } + return null; +} + function valueToEditText(value) { if (value === null || typeof value === "undefined") { return ""; } + if (isBase64JsonValue(value)) { + return value.encoded; + } if (typeof value === "object") { return JSON.stringify(value, null, 2); } @@ -3892,6 +4674,9 @@ function shouldUseTextarea(value, columnType) { if (columnType && columnType.type === "textarea") { return true; } + if (isBase64JsonValue(value)) { + return false; + } if (value && typeof value === "object") { return true; } @@ -3900,6 +4685,9 @@ function shouldUseTextarea(value, columnType) { } function rowEditValueKind(value) { + if (isBase64JsonValue(value)) { + return "binary"; + } if (value === null || typeof value === "undefined") { return "null"; } @@ -3923,6 +4711,261 @@ function rowEditControlElement(control, autocompleteUrl) { return autocomplete; } +function revokeRowEditBinaryPreview(wrapper) { + if (wrapper && wrapper._rowEditBinaryPreviewUrl) { + URL.revokeObjectURL(wrapper._rowEditBinaryPreviewUrl); + wrapper._rowEditBinaryPreviewUrl = null; + } +} + +function updateRowEditBinaryPreview(wrapper, encoded, byteLength) { + var preview = wrapper.querySelector(".row-edit-binary-preview"); + if (!preview) { + return; + } + revokeRowEditBinaryPreview(wrapper); + preview.hidden = true; + preview.textContent = ""; + if ( + !encoded || + byteLength >= ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES || + !window.atob || + !window.Blob || + !window.URL || + !URL.createObjectURL + ) { + return; + } + + var bytes; + try { + bytes = base64ToUint8Array(encoded); + } catch (_error) { + return; + } + var mimeType = rowEditBinaryImageMimeType(bytes); + if (!mimeType) { + return; + } + + var image = document.createElement("img"); + image.alt = ""; + var objectUrl = URL.createObjectURL(new Blob([bytes], { type: mimeType })); + wrapper._rowEditBinaryPreviewUrl = objectUrl; + + var showPreview = function () { + if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { + preview.hidden = false; + } + }; + var hidePreview = function () { + if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { + revokeRowEditBinaryPreview(wrapper); + preview.hidden = true; + preview.textContent = ""; + } + }; + image.onload = showPreview; + image.onerror = hidePreview; + image.src = objectUrl; + preview.appendChild(image); +} + +function updateRowEditBinaryDisplay(wrapper, control, fileName) { + var kind = rowEditControlValueKind(control); + var size = wrapper.querySelector(".row-edit-binary-size"); + var name = wrapper.querySelector(".row-edit-binary-name"); + var clear = wrapper.querySelector(".row-edit-binary-clear"); + var byteLength = + kind === "binary" ? binaryByteLengthFromBase64(control.value) : null; + + if (size) { + size.textContent = + kind === "binary" + ? formatRowEditBinarySize(byteLength) + : "No binary data"; + } + if (name) { + name.textContent = fileName || ""; + name.hidden = !fileName; + } + if (clear) { + clear.hidden = control.dataset.notNull === "1" || kind !== "binary"; + } + if (kind === "binary") { + updateRowEditBinaryPreview(wrapper, control.value, byteLength); + } else { + updateRowEditBinaryPreview(wrapper, "", 0); + } +} + +function setRowEditBinaryControlValue(control, encoded, fileName) { + control.value = encoded || ""; + control.dataset.currentValueKind = "binary"; + updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, fileName); + control.dispatchEvent(new Event("input", { bubbles: true })); +} + +function clearRowEditBinaryControlValue(control) { + control.value = ""; + control.dataset.currentValueKind = "null"; + updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, ""); + control.dispatchEvent(new Event("input", { bubbles: true })); +} + +function readRowEditBinaryFile(file) { + return new Promise(function (resolve, reject) { + var reader = new FileReader(); + reader.onload = function () { + try { + var bytes = new Uint8Array(reader.result || new ArrayBuffer(0)); + resolve({ + encoded: uint8ArrayToBase64(bytes), + name: file && file.name ? file.name : "", + }); + } catch (error) { + reject(error); + } + }; + reader.onerror = function () { + reject(reader.error || new Error("Could not read file")); + }; + reader.readAsArrayBuffer(file); + }); +} + +function rowEditBinaryValueFromText(text) { + var encoder = new TextEncoder(); + var bytes = encoder.encode(text || ""); + return { + encoded: uint8ArrayToBase64(bytes), + name: "Pasted text", + }; +} + +function rowEditBinaryFirstClipboardFile(clipboardData) { + if (!clipboardData) { + return null; + } + if (clipboardData.files && clipboardData.files.length) { + return clipboardData.files[0]; + } + var items = clipboardData.items || []; + for (var i = 0; i < items.length; i += 1) { + if (items[i].kind === "file") { + return items[i].getAsFile(); + } + } + return null; +} + +function handleRowEditBinaryFile(control, file) { + if (!file) { + return; + } + readRowEditBinaryFile(file) + .then(function (value) { + setRowEditBinaryControlValue(control, value.encoded, value.name); + }) + .catch(function (error) { + console.error("Could not read binary file", error); + }); +} + +function createRowEditBinaryControlElement(control, value, options, labelId) { + var wrapper = document.createElement("div"); + wrapper.className = "row-edit-binary-control"; + wrapper.dataset.column = control.name; + wrapper.setAttribute("role", "group"); + wrapper.setAttribute("tabindex", "0"); + wrapper.setAttribute("aria-labelledby", labelId); + + var preview = document.createElement("div"); + preview.className = "row-edit-binary-preview"; + preview.hidden = true; + + var status = document.createElement("div"); + status.className = "row-edit-binary-status"; + var size = document.createElement("span"); + size.className = "row-edit-binary-size"; + var name = document.createElement("span"); + name.className = "row-edit-binary-name"; + name.hidden = true; + status.appendChild(size); + status.appendChild(name); + + var actions = document.createElement("div"); + actions.className = "row-edit-binary-actions"; + var fileLabel = document.createElement("label"); + fileLabel.className = "row-edit-binary-file-button"; + fileLabel.textContent = "Attach file"; + var fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.setAttribute("aria-label", "Attach file for " + control.name); + fileInput.addEventListener("change", function () { + if (fileInput.files && fileInput.files.length) { + handleRowEditBinaryFile(control, fileInput.files[0]); + } + }); + fileLabel.appendChild(fileInput); + actions.appendChild(fileLabel); + + var clearButton = document.createElement("button"); + clearButton.type = "button"; + clearButton.className = "row-edit-binary-clear"; + clearButton.textContent = "Set NULL"; + clearButton.addEventListener("click", function () { + clearRowEditBinaryControlValue(control); + wrapper.focus(); + }); + actions.appendChild(clearButton); + + var dropTarget = document.createElement("div"); + dropTarget.className = "row-edit-binary-drop-target"; + dropTarget.textContent = "Drop or paste file contents"; + ["dragenter", "dragover"].forEach(function (eventName) { + wrapper.addEventListener(eventName, function (ev) { + ev.preventDefault(); + wrapper.classList.add("row-edit-binary-dragover"); + }); + }); + ["dragleave", "drop"].forEach(function (eventName) { + wrapper.addEventListener(eventName, function () { + wrapper.classList.remove("row-edit-binary-dragover"); + }); + }); + wrapper.addEventListener("drop", function (ev) { + ev.preventDefault(); + var file = ev.dataTransfer && ev.dataTransfer.files[0]; + handleRowEditBinaryFile(control, file); + }); + wrapper.addEventListener("paste", function (ev) { + var file = rowEditBinaryFirstClipboardFile(ev.clipboardData); + if (file) { + ev.preventDefault(); + handleRowEditBinaryFile(control, file); + return; + } + var text = ev.clipboardData && ev.clipboardData.getData("text"); + if (text) { + ev.preventDefault(); + var pasted = rowEditBinaryValueFromText(text); + setRowEditBinaryControlValue(control, pasted.encoded, pasted.name); + } + }); + + control.type = "hidden"; + control._rowEditBinaryWrapper = wrapper; + wrapper.appendChild(control); + wrapper.appendChild(preview); + wrapper.appendChild(status); + wrapper.appendChild(actions); + wrapper.appendChild(dropTarget); + + updateRowEditBinaryDisplay(wrapper, control, ""); + return wrapper; +} + function columnTypeForContext(columnType) { if (!columnType) { return null; @@ -4186,6 +5229,11 @@ function focusFirstRowEditControl(state, options) { if (focusRowEditPluginControl(field)) { return true; } + var binaryControl = field.querySelector(".row-edit-binary-control"); + if (binaryControl) { + binaryControl.focus(); + return true; + } control.focus(); return true; } @@ -4209,6 +5257,11 @@ function destroyRowEditFields(state) { } } }); + state.fields + .querySelectorAll(".row-edit-binary-control") + .forEach(function (binaryControl) { + revokeRowEditBinaryPreview(binaryControl); + }); state.fields.innerHTML = ""; } @@ -4235,34 +5288,50 @@ function createRowEditField(column, value, isPk, columnType, index, options) { controlWrap.className = "row-edit-control-wrap"; var context = columnFormControlContext(column, isPk, columnType, options); - var pluginControl = makeColumnField(options.manager, context); + var isBinaryField = shouldUseBinaryControl(value, options); + var pluginControl = isBinaryField + ? null + : makeColumnField(options.manager, context); var useTextarea = - (pluginControl && pluginControl.useTextarea === true) || - shouldUseTextarea(value, columnType); + !isBinaryField && + ((pluginControl && pluginControl.useTextarea === true) || + shouldUseTextarea(value, columnType)); var control = useTextarea ? document.createElement("textarea") : document.createElement("input"); + var initialValue = isBinaryField + ? binaryEncodedValue(value) + : valueToEditText(value); + var initialValueKind = options.valueKind || rowEditValueKind(value); + if (isBinaryField) { + initialValueKind = isBase64JsonValue(value) ? "binary" : "null"; + } control.className = "row-edit-input"; control.id = fieldId; control.name = column; - control.value = valueToEditText(value); + control.value = initialValue; control.setAttribute("aria-describedby", metaId); - control.dataset.initialValue = valueToEditText(value); - control.dataset.initialValueKind = - options.valueKind || rowEditValueKind(value); + control.dataset.initialValue = initialValue; + control.dataset.initialValueKind = initialValueKind; control.dataset.primaryKey = isPk ? "1" : "0"; control.dataset.currentValueKind = control.dataset.initialValueKind; + if (isBinaryField) { + control.dataset.binaryField = "1"; + control.dataset.notNull = options.notnull ? "1" : "0"; + } if (hasDefaultExpression) { control.dataset.useSqliteDefault = useSqliteDefault ? "1" : "0"; } if (useSqliteDefault) { control.disabled = true; } - if (options.omitIfBlank) { + if (options.omitIfBlank || (isBinaryField && options.mode === "insert")) { control.dataset.omitIfBlank = "1"; } - if (control.nodeName === "TEXTAREA") { + if (isBinaryField) { + control.type = "hidden"; + } else if (control.nodeName === "TEXTAREA") { control.rows = Math.min(8, Math.max(3, control.value.split("\n").length)); } else { control.type = "text"; @@ -4336,9 +5405,11 @@ function createRowEditField(column, value, isPk, columnType, index, options) { field._datasetteColumnFormField = fieldApi; var pluginControlElement = renderColumnField(pluginControl, fieldApi); var controlElement = + (isBinaryField && + createRowEditBinaryControlElement(control, value, options, labelId)) || pluginControlElement || rowEditControlElement(control, options.autocompleteUrl); - if (options.autocompleteUrl && !pluginControlElement) { + if (options.autocompleteUrl && !pluginControlElement && !isBinaryField) { control.addEventListener("input", function () { setForeignKeyMetaLink(meta, options.autocompleteUrl, null); }); @@ -4429,18 +5500,64 @@ function clearRowEditDialogError(state) { state.error.textContent = ""; } -function showRowEditDialogError(state, message) { +function showRowEditDialogError(state, message, options) { state.error.hidden = false; state.error.textContent = message; - state.error.focus(); + if (!options || options.focus !== false) { + state.error.focus(); + } +} + +function rowEditIsMultipleInsert(state) { + return state.mode === "insert" && state.insertMode === "multiple"; +} + +function syncRowEditInsertModeUi(state) { + var isInsert = state.mode === "insert"; + var isMultiple = rowEditIsMultipleInsert(state); + state.fields.hidden = isMultiple; + state.bulkInsertPanel.hidden = !isMultiple; + state.bulkInsertEditor.hidden = !isMultiple || state.bulkInsertPreviewReady; + state.bulkInsertPreview.hidden = !isMultiple || !state.bulkInsertPreviewReady; + state.bulkInsertLink.hidden = !isInsert || isMultiple; + state.singleInsertLink.hidden = !isInsert || !isMultiple; } function updateRowEditDialogButtons(state) { state.saveButton.disabled = - state.isLoading || state.isSaving || !state.hasLoaded; + state.isLoading || + state.isSaving || + !state.hasLoaded || + state.bulkInsertInserted || + (rowEditIsMultipleInsert(state) && + !state.bulkInsertPreviewReady && + !!state.bulkInsertLiveValidationError); state.cancelButton.disabled = state.isSaving; - var saveLabel = state.mode === "insert" ? "Insert row" : "Save"; - state.saveButton.textContent = state.isSaving ? "Saving..." : saveLabel; + syncRowEditInsertModeUi(state); + state.cancelButton.textContent = + rowEditIsMultipleInsert(state) && + state.bulkInsertPreviewReady && + !state.bulkInsertInserted + ? "Back" + : state.bulkInsertInserted + ? "Close and view table" + : "Cancel"; + var saveLabel = rowEditIsMultipleInsert(state) + ? state.bulkInsertInserted + ? "Inserted" + : state.bulkInsertPreviewReady + ? bulkInsertSaveLabel(state) + : "Preview rows" + : state.mode === "insert" + ? "Insert row" + : "Save"; + state.saveButton.textContent = state.isSaving + ? rowEditIsMultipleInsert(state) + ? bulkInsertConflictMode(state) === "upsert" + ? "Updating..." + : "Inserting..." + : "Saving..." + : saveLabel; state.form.setAttribute( "aria-busy", state.isLoading || state.isSaving ? "true" : "false", @@ -4455,11 +5572,15 @@ function setRowEditDialogLoading(state, isLoading) { function setRowEditDialogSaving(state, isSaving) { state.isSaving = isSaving; + state.modal.busy = isSaving; updateRowEditDialogButtons(state); } function valueFromRowEditControl(control) { var value = control.value; + if (rowEditControlValueKind(control) === "binary") { + return rowEditBinaryValue(value); + } return valueFromRowEditText( control.name, value, @@ -4470,6 +5591,9 @@ function valueFromRowEditControl(control) { function valueFromRowEditText(name, value, initialValueKind) { var trimmed = value.trim(); + if (initialValueKind === "binary") { + return rowEditBinaryValue(value); + } if (initialValueKind === "null" && value === "") { return null; } @@ -4597,7 +5721,11 @@ function collectRowFormValues(state) { if (control.dataset.useSqliteDefault === "1") { return; } - if (control.dataset.omitIfBlank === "1" && control.value === "") { + if ( + control.dataset.omitIfBlank === "1" && + control.value === "" && + rowEditControlValueKind(control) !== "binary" + ) { return; } if ( @@ -4631,6 +5759,16 @@ function rowEditDialogHasChanges(state) { if (!state || !state.hasLoaded || state.isLoading) { return false; } + if (state.bulkInsertInserted) { + return false; + } + if ( + state.mode === "insert" && + state.bulkInsertTextarea && + state.bulkInsertTextarea.value.trim() + ) { + return true; + } var fields = state.fields.querySelectorAll(".row-edit-field"); for (var i = 0; i < fields.length; i += 1) { var fieldApi = fields[i]._datasetteColumnFormField; @@ -4652,48 +5790,829 @@ function confirmDiscardRowEditChanges(state) { return window.confirm(message); } -function closeRowEditDialogIfConfirmed(state) { - if (!state || state.isSaving) { - return false; - } - if (!confirmDiscardRowEditChanges(state)) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; +function setRowInsertDialogTitle(state) { + var insertData = tableInsertData() || {}; + var title = rowEditIsMultipleInsert(state) + ? "Insert multiple rows" + : "Insert row"; + setRowDialogTitle( + state.title, + insertData.tableName ? title + " into " + insertData.tableName : title, + ); } -function scheduleCloseRowEditDialogIfConfirmed(state) { - // Fix for an issue in Safari where hitting Esc would show - // the confirm() prompt asking if state should be discarded - // but the Esc key press would then cancel that dialog too. - // Wait for keyup, then move the confirm() to a fresh timer tick. - if (!state || state.isSaving || state.isClosePending) { +function showMultipleRowInsert(state) { + if (!state || state.mode !== "insert" || state.isSaving) { + return; + } + state.insertMode = "multiple"; + if (state.bulkInsertPreviewReady || state.bulkInsertInserted) { + resetBulkInsertPreview(state); + } + clearRowEditDialogError(state); + setRowInsertDialogTitle(state); + syncBulkInsertConflictUi(state); + syncBulkInsertTextareaValidation(state); + updateRowEditDialogButtons(state); + state.bulkInsertTextarea.focus(); +} + +function showSingleRowInsert(state) { + if (!state || state.mode !== "insert" || state.isSaving) { + return; + } + state.insertMode = "single"; + clearRowEditDialogError(state); + setRowInsertDialogTitle(state); + updateRowEditDialogButtons(state); + if (!focusFirstRowEditControl(state, { skipReadonly: true })) { + state.saveButton.focus(); + } +} + +function bulkInsertConflictMode(state) { + if (!state || !state.bulkInsertHasPrimaryKeyColumns) { + return "insert"; + } + return state.bulkInsertConflictMode || "ignore"; +} + +function syncBulkInsertConflictUi(state) { + if (!state || !state.bulkInsertConflictField) { + return; + } + var insertData = tableInsertData() || {}; + var primaryKeys = insertData.primaryKeys || []; + var hasPrimaryKeys = primaryKeys.length > 0; + var hasPrimaryKeyColumns = + hasPrimaryKeys && + bulkInsertTextIncludesPrimaryKeyColumns( + state.bulkInsertTextarea.value, + state.bulkInsertColumnDetails, + primaryKeys, + ); + state.bulkInsertHasPrimaryKeyColumns = hasPrimaryKeyColumns; + var canUpsert = hasPrimaryKeyColumns && !!state.currentUpsertUrl; + var upsertOption = state.bulkInsertConflictSelect.querySelector( + 'option[value="upsert"]', + ); + if (upsertOption) { + upsertOption.disabled = !canUpsert; + upsertOption.hidden = !canUpsert; + } + if ( + !hasPrimaryKeyColumns || + (!canUpsert && bulkInsertConflictMode(state) === "upsert") + ) { + state.bulkInsertConflictMode = hasPrimaryKeys ? "ignore" : "insert"; + state.bulkInsertConflictSelect.value = state.bulkInsertConflictMode; + } + state.bulkInsertConflictField.hidden = !hasPrimaryKeyColumns; + state.bulkInsertConflictSelect.value = bulkInsertConflictMode(state); + var helpText = ""; + if (bulkInsertConflictMode(state) === "upsert") { + helpText = + "Rows with existing primary keys will be updated; new primary keys will be inserted."; + } else if (bulkInsertConflictMode(state) === "ignore") { + helpText = "Rows with existing primary keys will be skipped."; + } else { + helpText = "Rows with existing primary keys will stop the import."; + } + state.bulkInsertConflictHelp.textContent = helpText; +} + +function bulkInsertSaveLabel(state) { + if (!state.bulkInsertPreviewReady) { + return "Preview rows"; + } + if (bulkInsertConflictMode(state) === "upsert") { + return "Update or insert rows"; + } + return "Insert these rows"; +} + +function readTextFile(file) { + if (file.text) { + return file.text(); + } + return new Promise(function (resolve, reject) { + var reader = new FileReader(); + reader.onload = function () { + resolve(reader.result || ""); + }; + reader.onerror = function () { + reject(reader.error); + }; + reader.readAsText(file); + }); +} + +async function loadBulkInsertTextFile(state, file) { + if (!file) { + return; + } + try { + state.bulkInsertTextarea.value = await readTextFile(file); + state.bulkInsertTextarea.dispatchEvent( + new Event("input", { bubbles: true }), + ); + state.bulkInsertTextarea.focus(); + } catch (_error) { + showRowEditDialogError(state, "Could not read that text file."); + } +} + +function bulkInsertTemplateText(state) { + return (state.bulkInsertTemplateColumns || []).join("\t"); +} + +async function copyTextToClipboard(text) { + if (navigator.clipboard && navigator.clipboard.writeText) { + await navigator.clipboard.writeText(text); + return; + } + var textarea = document.createElement("textarea"); + textarea.value = text; + textarea.setAttribute("readonly", ""); + textarea.style.position = "fixed"; + textarea.style.top = "-1000px"; + textarea.style.left = "-1000px"; + document.body.appendChild(textarea); + textarea.select(); + var copied = document.execCommand("copy"); + textarea.remove(); + if (!copied) { + throw new Error("copy failed"); + } +} + +function setBulkInsertCopyButtonReady(state) { + state.copyTemplateButton.textContent = ""; + var wideLabel = document.createElement("span"); + wideLabel.className = "row-edit-copy-template-label-wide"; + wideLabel.textContent = "Copy spreadsheet template"; + state.copyTemplateButton.appendChild(wideLabel); + var narrowLabel = document.createElement("span"); + narrowLabel.className = "row-edit-copy-template-label-narrow"; + narrowLabel.textContent = "Copy template"; + state.copyTemplateButton.appendChild(narrowLabel); +} + +function setBulkInsertCopyButtonCopied(state) { + state.copyTemplateButton.textContent = "Copied"; + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = setTimeout(function () { + setBulkInsertCopyButtonReady(state); + }, 1500); +} + +function resetBulkInsertPreview(state) { + state.bulkInsertPreviewRows = null; + state.bulkInsertPreviewReady = false; + state.bulkInsertInserted = false; + state.bulkInsertInsertedCount = 0; + state.bulkInsertPreview.hidden = true; + state.bulkInsertPreview.textContent = ""; + state.bulkInsertProgress.hidden = true; + state.bulkInsertProgressBar.value = 0; + state.bulkInsertProgressBar.max = 1; + state.bulkInsertProgressStatus.textContent = ""; + syncBulkInsertConflictUi(state); + syncRowEditInsertModeUi(state); +} + +function normalizeBulkInsertCell(column, value) { + if (typeof value === "undefined") { + return column.notnull ? "" : null; + } + if (value === null) { + return column.notnull ? "" : null; + } + if (value === "" && column.notnull) { + return ""; + } + if (column.value_kind === "number" && typeof value === "string") { + return valueFromRowEditText(column.name, value, "number"); + } + return value; +} + +function rowObjectForBulkInsert(valuesByColumn, columns) { + var row = {}; + columns.forEach(function (column) { + var hasValue = Object.prototype.hasOwnProperty.call( + valuesByColumn, + column.name, + ); + if (!hasValue) { + return; + } + row[column.name] = normalizeBulkInsertCell( + column, + valuesByColumn[column.name], + ); + }); + return row; +} + +function splitDelimitedRows(text, delimiter) { + var rows = []; + var row = []; + var cell = ""; + var inQuotes = false; + + for (var i = 0; i < text.length; i += 1) { + var character = text[i]; + if (inQuotes) { + if (character === '"') { + if (text[i + 1] === '"') { + cell += '"'; + i += 1; + } else { + inQuotes = false; + } + } else { + cell += character; + } + continue; + } + + if (character === '"') { + inQuotes = true; + } else if (character === delimiter) { + row.push(cell); + cell = ""; + } else if (character === "\n" || character === "\r") { + row.push(cell); + rows.push(row); + row = []; + cell = ""; + if (character === "\r" && text[i + 1] === "\n") { + i += 1; + } + } else { + cell += character; + } + } + + if (inQuotes) { + throw new Error("Unclosed quoted value."); + } + row.push(cell); + rows.push(row); + + while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { + rows.pop(); + } + return rows; +} + +function bulkInsertDelimitedRowIsBlank(row) { + return row.every(function (value) { + return value.trim() === ""; + }); +} + +function delimiterPreviewRows(text, delimiter) { + try { + return splitDelimitedRows(text, delimiter); + } catch (_error) { + return []; + } +} + +function splitSingleColumnRows(text) { + var rows = text.split(/\r\n|\n|\r/).map(function (line) { + return [line]; + }); + while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { + rows.pop(); + } + return rows; +} + +function detectBulkInsertDelimiter(text, columns) { + var firstLine = + text.split(/\r\n|\n|\r/).find(function (line) { + return line.trim() !== ""; + }) || ""; + var csvRows = delimiterPreviewRows(firstLine, ","); + var tsvRows = delimiterPreviewRows(firstLine, "\t"); + var csvColumns = csvRows.length ? csvRows[0].length : 0; + var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; + + if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { + return "\t"; + } + if (tsvColumns > csvColumns) { + return "\t"; + } + if (csvColumns > 1) { + return ","; + } + if (tsvColumns > 1) { + return "\t"; + } + if (columns.length === 1 || bulkInsertColumnMap(columns)[firstLine.trim()]) { + return null; + } + throw new Error("Could not detect CSV or TSV columns."); +} + +function bulkInsertColumnMap(columns) { + var map = {}; + columns.forEach(function (column) { + map[column.name] = column; + }); + return map; +} + +function bulkInsertTextIncludesPrimaryKeyColumns(text, columns, primaryKeys) { + if (!primaryKeys.length || !text.trim()) { return false; } - if (!rowEditDialogHasChanges(state)) { - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; + var trimmed = text.trim(); + try { + if (trimmed[0] === "[" || trimmed[0] === "{") { + return jsonBulkInsertTextIncludesPrimaryKeyColumns(trimmed, primaryKeys); + } + return delimitedBulkInsertTextIncludesPrimaryKeyColumns( + trimmed, + columns, + primaryKeys, + ); + } catch (_error) { + return false; } - state.isClosePending = true; - var closeAfterKeyup = function () { - if (!state.isClosePending) { - return; +} + +function jsonBulkInsertTextIncludesPrimaryKeyColumns(text, primaryKeys) { + var rows = parseJsonObjectRows(text); + var seenKeys = {}; + rows.forEach(function (row) { + Object.keys(row).forEach(function (key) { + seenKeys[key] = true; + }); + }); + return primaryKeys.every(function (key) { + return !!seenKeys[key]; + }); +} + +function delimitedBulkInsertTextIncludesPrimaryKeyColumns( + text, + columns, + primaryKeys, +) { + var delimiter = detectBulkInsertDelimiter(text, columns); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + return false; + } + + var columnMap = bulkInsertColumnMap(columns); + var header = rows[0].map(function (value) { + return value.trim(); + }); + var headerMatches = header.filter(function (name) { + return !!columnMap[name]; + }).length; + if (headerMatches > 0) { + return primaryKeys.every(function (key) { + return header.indexOf(key) !== -1; + }); + } + + var headers = columns.map(function (column) { + return column.name; + }); + var suppliedColumnCount = rows.reduce(function (count, row) { + return Math.max(count, row.length); + }, 0); + return primaryKeys.every(function (key) { + var index = headers.indexOf(key); + return index !== -1 && index < suppliedColumnCount; + }); +} + +function bulkInsertLiveValidationShouldWait(message) { + return ( + message === "Paste rows before previewing." || + message === "No data rows found to preview." || + message.indexOf("Invalid JSON:") === 0 + ); +} + +function bulkInsertLiveValidationError(state) { + var text = state.bulkInsertTextarea.value; + if (!text.trim()) { + return null; + } + try { + parseBulkInsertRows(text, state.bulkInsertColumnDetails); + } catch (error) { + var message = error.message || "Could not preview rows."; + return bulkInsertLiveValidationShouldWait(message) ? null : message; + } + return null; +} + +function syncBulkInsertTextareaValidation(state) { + if (!rowEditIsMultipleInsert(state) || state.bulkInsertPreviewReady) { + state.bulkInsertLiveValidationError = null; + return; + } + state.bulkInsertLiveValidationError = bulkInsertLiveValidationError(state); + if (state.bulkInsertLiveValidationError) { + showRowEditDialogError(state, state.bulkInsertLiveValidationError, { + focus: false, + }); + } else { + clearRowEditDialogError(state); + } +} + +function parseJsonBulkInsertRows(text, columns) { + var parsed = parseJsonObjectRows(text); + + var columnMap = bulkInsertColumnMap(columns); + return parsed.map(function (item, index) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("JSON row " + (index + 1) + " must be an object."); } - state.isClosePending = false; - closeRowEditDialogIfConfirmed(state); - }; - var onKeyup = function (ev) { - if (ev.key !== "Escape") { - return; + Object.keys(item).forEach(function (key) { + if (!columnMap[key]) { + throw new Error( + "JSON row " + (index + 1) + " has unknown column " + key + ".", + ); + } + }); + return rowObjectForBulkInsert(item, columns); + }); +} + +function parseDelimitedBulkInsertRows(text, columns) { + var delimiter = detectBulkInsertDelimiter(text, columns); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + + var columnMap = bulkInsertColumnMap(columns); + var header = rows[0].map(function (value) { + return value.trim(); + }); + var headerMatches = header.filter(function (name) { + return !!columnMap[name]; + }).length; + var hasHeader = headerMatches > 0; + var dataRows = hasHeader ? rows.slice(1) : rows; + var headers = hasHeader + ? header + : columns.map(function (column) { + return column.name; + }); + var seenHeaders = {}; + + if (hasHeader) { + headers.forEach(function (name) { + if (!name) { + return; + } + if (!columnMap[name]) { + throw new Error("Unknown column " + name + " in header row."); + } + if (seenHeaders[name]) { + throw new Error("Duplicate column " + name + " in header row."); + } + seenHeaders[name] = true; + }); + } + + if (!dataRows.length) { + throw new Error("No data rows found to preview."); + } + + return dataRows.map(function (row, rowIndex) { + if (row.length > headers.length) { + throw new Error( + "Row " + + (rowIndex + 1) + + " has " + + row.length + + " values, but only " + + headers.length + + " columns were provided.", + ); } - document.removeEventListener("keyup", onKeyup, true); - setTimeout(closeAfterKeyup, 0); + var valuesByColumn = {}; + row.forEach(function (value, index) { + var columnName = headers[index]; + if (columnMap[columnName]) { + valuesByColumn[columnName] = value; + } + }); + return rowObjectForBulkInsert(valuesByColumn, columns); + }); +} + +function parseBulkInsertRows(text, columns) { + var trimmed = text.trim(); + if (!trimmed) { + throw new Error("Paste rows before previewing."); + } + if (trimmed[0] === "[" || trimmed[0] === "{") { + return parseJsonBulkInsertRows(trimmed, columns); + } + return parseDelimitedBulkInsertRows(trimmed, columns); +} + +function bulkInsertPreviewValue(value) { + if (value === null) { + return "null"; + } + if (typeof value === "object") { + return JSON.stringify(value); + } + return String(value); +} + +function bulkInsertPreviewCell(column, hasValue, value) { + if (!hasValue && column.is_auto_pk) { + return { + text: "auto", + className: "row-edit-bulk-preview-auto", + }; + } + if (value === null) { + return { + text: bulkInsertPreviewValue(value), + className: "row-edit-bulk-preview-null", + }; + } + return { + text: hasValue ? bulkInsertPreviewValue(value) : "", + className: "", }; - document.addEventListener("keyup", onKeyup, true); - return true; +} + +function renderBulkInsertPreview(state, rows) { + state.bulkInsertPreview.textContent = ""; + var summary = document.createElement("p"); + summary.className = "row-edit-bulk-preview-summary"; + summary.textContent = + "Previewing " + rows.length + " row" + (rows.length === 1 ? "." : "s."); + state.bulkInsertPreview.appendChild(summary); + + var tableWrap = document.createElement("div"); + tableWrap.className = "row-edit-bulk-preview-table-wrap"; + var table = document.createElement("table"); + table.className = "row-edit-bulk-preview-table"; + var thead = document.createElement("thead"); + var headerRow = document.createElement("tr"); + state.bulkInsertColumnDetails.forEach(function (column) { + var th = document.createElement("th"); + th.scope = "col"; + th.textContent = column.name; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + var tbody = document.createElement("tbody"); + rows.forEach(function (row) { + var tr = document.createElement("tr"); + state.bulkInsertColumnDetails.forEach(function (column) { + var td = document.createElement("td"); + var hasValue = Object.prototype.hasOwnProperty.call(row, column.name); + var value = hasValue ? row[column.name] : ""; + var cell = bulkInsertPreviewCell(column, hasValue, value); + td.textContent = cell.text; + if (cell.className) { + td.className = cell.className; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + tableWrap.appendChild(table); + state.bulkInsertPreview.appendChild(tableWrap); + state.bulkInsertPreview.hidden = false; +} + +function previewBulkInsertRows(state) { + clearRowEditDialogError(state); + resetBulkInsertPreview(state); + syncBulkInsertConflictUi(state); + try { + var rows = parseBulkInsertRows( + state.bulkInsertTextarea.value, + state.bulkInsertColumnDetails, + ); + state.bulkInsertPreviewRows = rows; + state.bulkInsertPreviewReady = true; + renderBulkInsertPreview(state, rows); + updateRowEditDialogButtons(state); + } catch (error) { + showRowEditDialogError(state, error.message || "Could not preview rows."); + updateRowEditDialogButtons(state); + } +} + +function updateBulkInsertProgress(state, inserted, total) { + var words = bulkInsertProgressWords(state); + state.bulkInsertProgress.hidden = false; + state.bulkInsertProgressBar.max = total || 1; + state.bulkInsertProgressBar.value = inserted; + state.bulkInsertProgressStatus.textContent = + inserted >= total + ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." + : words.active + " " + inserted + " of " + total + " rows..."; +} + +function bulkInsertBatches(rows, batchSize) { + var batches = []; + var size = Math.max(1, batchSize || 1); + for (var index = 0; index < rows.length; index += size) { + batches.push(rows.slice(index, index + size)); + } + return batches; +} + +function animateBulkInsertProgress(state, from, to, total, duration) { + state.bulkInsertProgress.hidden = false; + state.bulkInsertProgressBar.max = total || 1; + if (duration <= 0 || !window.requestAnimationFrame) { + updateBulkInsertProgress(state, to, total); + return Promise.resolve(); + } + + return new Promise(function (resolve) { + var startTime = null; + var step = function (timestamp) { + if (startTime === null) { + startTime = timestamp; + } + var progress = Math.min((timestamp - startTime) / duration, 1); + var easedProgress = 1 - Math.pow(1 - progress, 3); + var value = from + (to - from) * easedProgress; + var displayValue = progress === 1 ? to : Math.floor(value); + var words = bulkInsertProgressWords(state); + state.bulkInsertProgressBar.value = value; + state.bulkInsertProgressStatus.textContent = + displayValue >= total + ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." + : words.active + " " + displayValue + " of " + total + " rows..."; + if (progress < 1) { + window.requestAnimationFrame(step); + } else { + updateBulkInsertProgress(state, to, total); + resolve(); + } + }; + window.requestAnimationFrame(step); + }); +} + +function bulkInsertProgressWords(state) { + var mode = bulkInsertConflictMode(state); + if (mode === "upsert") { + return { + active: "Upserting", + complete: "upserted", + }; + } + if (mode === "ignore") { + return { + active: "Processing", + complete: "processed", + }; + } + return { + active: "Inserting", + complete: "inserted", + }; +} + +function validateBulkInsertConflictRows(state, rows) { + if (bulkInsertConflictMode(state) !== "upsert") { + return null; + } + var insertData = tableInsertData() || {}; + var primaryKeys = insertData.primaryKeys || []; + for (var index = 0; index < rows.length; index += 1) { + var row = rows[index]; + var missing = primaryKeys.filter(function (key) { + return ( + !Object.prototype.hasOwnProperty.call(row, key) || + row[key] === null || + typeof row[key] === "undefined" + ); + }); + if (missing.length) { + return ( + "Row " + + (index + 1) + + " is missing primary key " + + missing.join(", ") + + ". Upsert requires primary key values for every row." + ); + } + } + return null; +} + +async function insertBulkPreviewRows(state) { + if (!state.bulkInsertPreviewRows || state.bulkInsertInserted) { + return; + } + var conflictMode = bulkInsertConflictMode(state); + var url = + conflictMode === "upsert" ? state.currentUpsertUrl : state.currentInsertUrl; + if (!url) { + showRowEditDialogError( + state, + conflictMode === "upsert" + ? "Could not find the row upsert URL." + : "Could not find the row insert URL.", + ); + return; + } + + var rows = state.bulkInsertPreviewRows; + var validationError = validateBulkInsertConflictRows(state, rows); + if (validationError) { + showRowEditDialogError(state, validationError); + return; + } + var total = rows.length; + var inserted = state.bulkInsertInsertedCount || 0; + var batches = bulkInsertBatches( + rows.slice(inserted), + state.bulkInsertMaxRows, + ); + var progressAnimationDuration = 500 / Math.max(batches.length, 1); + + clearRowEditDialogError(state); + updateBulkInsertProgress(state, inserted, total); + setRowEditDialogSaving(state, true); + try { + for (var batchIndex = 0; batchIndex < batches.length; batchIndex += 1) { + var batch = batches[batchIndex]; + var payload = { rows: batch }; + if (conflictMode === "ignore") { + payload.ignore = true; + } + var response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = null; + try { + data = await response.json(); + } catch (_error) { + data = null; + } + if (!response.ok || (data && data.ok === false)) { + throw rowMutationRequestError(response, data); + } + var previousInserted = inserted; + inserted += batch.length; + state.bulkInsertInsertedCount = inserted; + await animateBulkInsertProgress( + state, + previousInserted, + inserted, + total, + progressAnimationDuration, + ); + } + state.bulkInsertInserted = true; + state.shouldReloadOnClose = true; + state.redirectOnCloseUrl = tableBaseUrl().toString(); + updateBulkInsertProgress(state, inserted, total); + } catch (error) { + showRowEditDialogError(state, error.message || "Could not insert rows."); + } finally { + setRowEditDialogSaving(state, false); + } } function findDataRowElement(root, rowId) { @@ -4762,6 +6681,14 @@ async function saveRowEditDialog(state) { if (state.isLoading || state.isSaving || !state.hasLoaded) { return; } + if (rowEditIsMultipleInsert(state)) { + if (!state.bulkInsertPreviewReady) { + previewBulkInsertRows(state); + } else if (!state.bulkInsertInserted) { + await insertBulkPreviewRows(state); + } + return; + } clearRowEditDialogError(state); setRowEditDialogSaving(state, true); @@ -4777,9 +6704,8 @@ async function saveRowEditDialog(state) { } var formValues = collectRowFormValues(state); if (state.mode === "edit" && !Object.keys(formValues).length) { - state.shouldRestoreFocus = true; hideRowMutationStatus(); - state.dialog.close(); + state.modal.close(); return; } var payload = @@ -4812,9 +6738,8 @@ async function saveRowEditDialog(state) { insertedRowData, insertData.primaryKeys || [], ); - state.shouldRestoreFocus = false; if (!insertedRowId) { - state.dialog.close(); + state.modal.close({ restoreFocus: false }); var missingIdStatus = showRowMutationStatus( state.manager, "Inserted row. Refresh the page to see it.", @@ -4830,7 +6755,7 @@ async function saveRowEditDialog(state) { try { insertedRow = await fetchUpdatedRowElement(state); } catch (_error) { - state.dialog.close(); + state.modal.close({ restoreFocus: false }); var refreshFailedStatus = showRowMutationStatus( state.manager, "Inserted row, but could not refresh the table row. Refresh the page to see it.", @@ -4845,7 +6770,7 @@ async function saveRowEditDialog(state) { rowTitleLabel(insertedRow), ); var addedRow = addInsertedRowToPage(insertedRow); - state.dialog.close(); + state.modal.close({ restoreFocus: false }); showRowMutationStatus(state.manager, insertedStatusMessage, false); if (addedRow) { var insertedFocusTarget = @@ -4854,7 +6779,7 @@ async function saveRowEditDialog(state) { insertedFocusTarget.focus(); } } else { - state.dialog.close(); + state.modal.close({ restoreFocus: false }); var filteredStatus = showRowMutationStatus( state.manager, "Inserted row. It does not match the current filters.", @@ -4866,8 +6791,7 @@ async function saveRowEditDialog(state) { } if (isRowPage()) { - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); location.reload(); return; } @@ -4903,8 +6827,7 @@ async function saveRowEditDialog(state) { ); } - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (focusTarget && document.contains(focusTarget)) { focusTarget.focus(); } @@ -4919,9 +6842,12 @@ function renderRowEditFields(state, data) { var columns = data.columns || (row ? Object.keys(row) : []); var primaryKeys = data.primary_keys || []; var columnTypes = data.column_types || {}; + var columnDetails = data.column_details || {}; + state.insertMode = "single"; destroyRowEditFields(state); columns.forEach(function (column, index) { + var columnDetail = columnDetails[column] || {}; state.fields.appendChild( createRowEditField( column, @@ -4935,7 +6861,9 @@ function renderRowEditFields(state, data) { form: state.form, manager: state.manager, mode: state.mode, + notnull: columnDetail.notnull, primaryKeyReadonly: true, + sqliteType: columnDetail.sqlite_type, }, ), ); @@ -4950,7 +6878,23 @@ function renderRowEditFields(state, data) { function renderRowInsertFields(state, data) { var columns = data.columns || []; + var bulkColumns = data.bulkColumns || columns; + state.insertMode = "single"; + state.bulkInsertColumnDetails = bulkColumns.slice(); + state.bulkInsertMaxRows = data.maxInsertRows || 100; + state.bulkInsertColumns = bulkColumns.map(function (column) { + return column.name; + }); + state.bulkInsertTemplateColumns = columns.map(function (column) { + return column.name; + }); + state.copyTemplateButton.disabled = !state.bulkInsertTemplateColumns.length; + setBulkInsertCopyButtonReady(state); + syncBulkInsertConflictUi(state); + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = null; + resetBulkInsertPreview(state); destroyRowEditFields(state); columns.forEach(function (column, index) { state.fields.appendChild( @@ -5027,7 +6971,8 @@ function ensureRowEditDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = ROW_EDIT_DIALOG_ID; dialog.className = "row-edit-dialog"; dialog.setAttribute("aria-labelledby", "row-edit-title"); @@ -5039,16 +6984,46 @@ function ensureRowEditDialog(manager) {

      Loading row...

      -
      +
      +
      - - + + + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); rowEditDialogState = { + modal: modal, dialog: dialog, form: dialog.querySelector(".row-edit-form"), title: dialog.querySelector(".modal-title"), @@ -5056,23 +7031,57 @@ function ensureRowEditDialog(manager) { loading: dialog.querySelector(".row-edit-loading"), error: dialog.querySelector(".row-edit-error"), fields: dialog.querySelector(".row-edit-fields"), + bulkInsertPanel: dialog.querySelector(".row-edit-bulk"), + bulkInsertEditor: dialog.querySelector(".row-edit-bulk-editor"), + bulkInsertTextarea: dialog.querySelector(".row-edit-bulk-textarea"), + bulkInsertPreview: dialog.querySelector(".row-edit-bulk-preview"), + bulkInsertProgress: dialog.querySelector(".row-edit-bulk-progress"), + bulkInsertProgressBar: dialog.querySelector(".row-edit-bulk-progress-bar"), + bulkInsertProgressStatus: dialog.querySelector( + ".row-edit-bulk-progress-status", + ), + bulkInsertConflictField: dialog.querySelector(".row-edit-bulk-conflict"), + bulkInsertConflictSelect: dialog.querySelector( + ".row-edit-bulk-conflict-mode", + ), + bulkInsertConflictHelp: dialog.querySelector( + ".row-edit-bulk-conflict-help", + ), + copyTemplateButton: dialog.querySelector(".row-edit-copy-template"), + bulkInsertOpenFileButton: dialog.querySelector(".row-edit-bulk-open-file"), + bulkInsertFileInput: dialog.querySelector(".row-edit-bulk-file-input"), + bulkInsertLink: dialog.querySelector(".row-edit-bulk-insert"), + singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), - currentButton: null, currentRow: null, currentRowId: null, currentPkPath: null, currentInsertUrl: null, + currentUpsertUrl: null, currentUpdateUrl: null, currentFragmentUrl: null, mode: "edit", + insertMode: "single", + bulkInsertConflictMode: "ignore", + bulkInsertHasPrimaryKeyColumns: false, + bulkInsertLiveValidationError: null, + bulkInsertColumns: [], + bulkInsertTemplateColumns: [], + bulkInsertColumnDetails: [], + bulkInsertPreviewRows: null, + bulkInsertPreviewReady: false, + bulkInsertInserted: false, + bulkInsertInsertedCount: 0, + bulkInsertMaxRows: 100, + shouldReloadOnClose: false, + redirectOnCloseUrl: null, + copyTemplateResetTimer: null, loadId: 0, manager: manager, isLoading: false, isSaving: false, - isClosePending: false, hasLoaded: false, - shouldRestoreFocus: true, }; rowEditDialogState.form.addEventListener("submit", function (ev) { @@ -5081,46 +7090,165 @@ function ensureRowEditDialog(manager) { }); rowEditDialogState.cancelButton.addEventListener("click", function () { - if (!rowEditDialogState.isSaving) { - rowEditDialogState.shouldRestoreFocus = true; - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeRowEditDialogIfConfirmed(rowEditDialogState); - } - }); - - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { + if ( + rowEditIsMultipleInsert(rowEditDialogState) && + rowEditDialogState.bulkInsertPreviewReady && + !rowEditDialogState.bulkInsertInserted && + !rowEditDialogState.isSaving + ) { + resetBulkInsertPreview(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + rowEditDialogState.bulkInsertTextarea.focus(); return; } - ev.preventDefault(); - scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); + modal.requestClose("cancel"); }); - dialog.addEventListener("cancel", function (ev) { + rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { ev.preventDefault(); - scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); + showMultipleRowInsert(rowEditDialogState); }); + rowEditDialogState.singleInsertLink.addEventListener("click", function (ev) { + ev.preventDefault(); + showSingleRowInsert(rowEditDialogState); + }); + + rowEditDialogState.copyTemplateButton.addEventListener( + "click", + async function () { + try { + await copyTextToClipboard(bulkInsertTemplateText(rowEditDialogState)); + clearRowEditDialogError(rowEditDialogState); + setBulkInsertCopyButtonCopied(rowEditDialogState); + } catch (_error) { + showRowEditDialogError( + rowEditDialogState, + "Could not copy the spreadsheet template.", + ); + } + }, + ); + + rowEditDialogState.bulkInsertOpenFileButton.addEventListener( + "click", + function () { + rowEditDialogState.bulkInsertFileInput.click(); + }, + ); + + rowEditDialogState.bulkInsertFileInput.addEventListener( + "change", + async function (ev) { + var files = ev.target.files; + await loadBulkInsertTextFile( + rowEditDialogState, + files && files.length ? files[0] : null, + ); + ev.target.value = ""; + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragenter", + function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.add( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragover", + function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.add( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragleave", + function () { + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "drop", + async function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + var files = ev.dataTransfer && ev.dataTransfer.files; + if (!files || !files.length) { + return; + } + await loadBulkInsertTextFile(rowEditDialogState, files[0]); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragend", + function () { + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener("input", function () { + resetBulkInsertPreview(rowEditDialogState); + syncBulkInsertTextareaValidation(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + }); + + rowEditDialogState.bulkInsertConflictSelect.addEventListener( + "change", + function () { + rowEditDialogState.bulkInsertConflictMode = + rowEditDialogState.bulkInsertConflictSelect.value; + syncBulkInsertConflictUi(rowEditDialogState); + resetBulkInsertPreview(rowEditDialogState); + syncBulkInsertTextareaValidation(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + }, + ); + + modal.beforeClose = function (source) { + return ( + source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState) + ); + }; + dialog.addEventListener("close", function () { var state = rowEditDialogState; + var shouldReloadOnClose = state.shouldReloadOnClose; + var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; - state.isClosePending = false; + state.bulkInsertLiveValidationError = null; + state.shouldReloadOnClose = false; + state.redirectOnCloseUrl = null; + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = null; + setBulkInsertCopyButtonReady(state); + resetBulkInsertPreview(state); clearRowEditDialogError(state); state.hasLoaded = false; destroyRowEditFields(state); setRowEditDialogLoading(state, false); setRowEditDialogSaving(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); + if (shouldReloadOnClose) { + if (redirectOnCloseUrl) { + location.href = redirectOnCloseUrl; + } else { + location.reload(); + } } }); @@ -5139,13 +7267,14 @@ async function openRowEditDialog(button, manager) { state.manager = manager; state.mode = "edit"; - state.currentButton = button; state.currentRow = row; state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); state.currentInsertUrl = null; + state.currentUpsertUrl = null; state.currentUpdateUrl = rowUpdateUrl(row); state.currentFragmentUrl = rowFragmentUrl(row); + state.insertMode = "single"; if (state.currentUpdateUrl) { state.form.action = new URL( state.currentUpdateUrl, @@ -5154,7 +7283,7 @@ async function openRowEditDialog(button, manager) { } else { state.form.removeAttribute("action"); } - state.shouldRestoreFocus = true; + state.hasLoaded = false; state.loadId += 1; var loadId = state.loadId; @@ -5171,10 +7300,9 @@ async function openRowEditDialog(button, manager) { ); state.summary.hidden = true; state.summary.textContent = ""; + syncRowEditInsertModeUi(state); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); state.cancelButton.focus(); try { @@ -5214,14 +7342,21 @@ function openRowInsertDialog(button, manager) { state.manager = manager; state.mode = "insert"; - state.currentButton = button; state.currentRow = null; state.currentRowId = null; state.currentPkPath = null; state.currentInsertUrl = tableInsertUrl(); + state.currentUpsertUrl = tableUpsertUrl(); state.currentUpdateUrl = null; state.currentFragmentUrl = null; - state.shouldRestoreFocus = true; + state.insertMode = "single"; + state.bulkInsertConflictMode = "ignore"; + state.bulkInsertLiveValidationError = null; + state.bulkInsertTextarea.value = ""; + state.shouldReloadOnClose = false; + state.redirectOnCloseUrl = null; + resetBulkInsertPreview(state); + state.hasLoaded = false; state.loadId += 1; @@ -5238,18 +7373,12 @@ function openRowInsertDialog(button, manager) { setRowEditDialogLoading(state, false); destroyRowEditFields(state); state.dialog.removeAttribute("aria-describedby"); - setRowDialogTitle( - state.title, - insertData.tableName - ? "Insert row into " + insertData.tableName - : "Insert row", - ); + setRowInsertDialogTitle(state); state.summary.hidden = true; state.summary.textContent = ""; + syncRowEditInsertModeUi(state); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); renderRowInsertFields(state, insertData); } diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js deleted file mode 100644 index 0e6e2c29..00000000 --- a/datasette/static/json-format-highlight-1.0.1.js +++ /dev/null @@ -1,56 +0,0 @@ -/* -https://github.com/luyilin/json-format-highlight -From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js -MIT Licensed -*/ -(function (global, factory) { - typeof exports === "object" && typeof module !== "undefined" - ? (module.exports = factory()) - : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); -})(this, function () { - "use strict"; - - var defaultColors = { - keyColor: "dimgray", - numberColor: "lightskyblue", - stringColor: "lightcoral", - trueColor: "lightseagreen", - falseColor: "#f66578", - nullColor: "cornflowerblue", - }; - - function index(json, colorOptions) { - if (colorOptions === void 0) colorOptions = {}; - - if (!json) { - return; - } - if (typeof json !== "string") { - json = JSON.stringify(json, null, 2); - } - var colors = Object.assign({}, defaultColors, colorOptions); - json = json.replace(/&/g, "&").replace(//g, ">"); - return json.replace( - /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, - function (match) { - var color = colors.numberColor; - if (/^"/.test(match)) { - color = /:$/.test(match) ? colors.keyColor : colors.stringColor; - } else { - color = /true/.test(match) - ? colors.trueColor - : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; - } - return '' + match + ""; - }, - ); - } - - return index; -}); diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js index a386b1fc..e94ee865 100644 --- a/datasette/static/mobile-column-actions.js +++ b/datasette/static/mobile-column-actions.js @@ -66,7 +66,8 @@ function initMobileColumnActions(manager) { return; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.className = "mobile-column-actions-dialog"; dialog.id = MOBILE_COLUMN_DIALOG_ID; dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); @@ -75,13 +76,13 @@ function initMobileColumnActions(manager) { Column actions
      -
      +
      Tap a column to reveal actions. - +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); triggerButton.setAttribute("aria-haspopup", "dialog"); triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); @@ -91,7 +92,6 @@ function initMobileColumnActions(manager) { var listWrap = dialog.querySelector(".mobile-column-list"); var doneButton = dialog.querySelector(".mobile-column-actions-done"); var expandedSectionId = null; - var shouldRestoreFocus = true; function updateExpandedSection() { Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { @@ -128,16 +128,7 @@ function initMobileColumnActions(manager) { } function closeDialog(options) { - options = options || {}; - shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } - } + modal.close(options); } function renderDialog() { @@ -166,7 +157,8 @@ function initMobileColumnActions(manager) { topActions.className = "mobile-column-top-actions"; var showAllColumns = document.createElement("a"); - showAllColumns.className = "btn btn-ghost mobile-column-top-action"; + showAllColumns.className = + "modal-btn modal-btn-ghost mobile-column-top-action"; showAllColumns.href = manager.columnActions.showAllColumnsUrl(); showAllColumns.textContent = "Show all columns"; @@ -265,9 +257,7 @@ function initMobileColumnActions(manager) { if (!renderDialog()) { return; } - if (!dialog.open) { - dialog.showModal(); - } + modal.show({ returnFocusTo: triggerButton }); triggerButton.setAttribute("aria-expanded", "true"); var focusTarget = dialog.querySelector(".mobile-column-top-action") || @@ -288,22 +278,8 @@ function initMobileColumnActions(manager) { closeDialog(); }); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeDialog(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeDialog(); - }); - dialog.addEventListener("close", function () { triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } }); window.addEventListener("resize", function () { diff --git a/datasette/static/modal.js b/datasette/static/modal.js new file mode 100644 index 00000000..ca9ab353 --- /dev/null +++ b/datasette/static/modal.js @@ -0,0 +1,167 @@ +// Shared lifecycle for native modal dialogs. +(() => { + class DatasetteModal extends HTMLElement { + constructor() { + super(); + this.beforeClose = null; + this._busy = false; + this._restoreFocus = true; + this._returnFocusTo = null; + this._escapeCleanup = null; + this._escapeTimer = null; + } + + static create() { + const modal = document.createElement("datasette-modal"); + modal.appendChild(document.createElement("dialog")); + return modal; + } + + get dialog() { + return this.querySelector(":scope > dialog"); + } + + get busy() { + return this._busy; + } + + set busy(value) { + this._busy = !!value; + if (this.dialog) { + this.dialog.setAttribute("aria-busy", String(this._busy)); + } + } + + connectedCallback() { + const dialog = this.dialog; + if (!dialog) return; + dialog.classList.add("datasette-modal"); + this._listeners?.abort(); + this._listeners = new AbortController(); + const options = { signal: this._listeners.signal }; + let backdropPointerDown = false; + const outside = (event) => { + const rect = dialog.getBoundingClientRect(); + return ( + event.target === dialog && + (event.clientX < rect.left || + event.clientX > rect.right || + event.clientY < rect.top || + event.clientY > rect.bottom) + ); + }; + dialog.addEventListener( + "pointerdown", + (event) => { + backdropPointerDown = outside(event); + }, + options, + ); + dialog.addEventListener( + "click", + (event) => { + if (backdropPointerDown && outside(event)) + this.requestClose("backdrop"); + backdropPointerDown = false; + }, + options, + ); + dialog.addEventListener( + "keydown", + (event) => { + if (event.key !== "Escape" || event.defaultPrevented) return; + // A nested native dialog or plugin picker gets first refusal. + if (event.target.closest("dialog") !== dialog) return; + event.preventDefault(); + if (this.busy || this._escapeCleanup || this._escapeTimer !== null) + return; + // Safari can otherwise use this Escape press to cancel confirm() too. + // Only keyboard dismissals wait for keyup; native cancel events needn't. + const onKeyup = (up) => { + if (up.key !== "Escape") return; + this._escapeCleanup(); + this._escapeCleanup = null; + this._escapeTimer = setTimeout(() => { + this._escapeTimer = null; + this.requestClose("escape"); + }, 0); + }; + this.ownerDocument.addEventListener("keyup", onKeyup, true); + this._escapeCleanup = () => + this.ownerDocument.removeEventListener("keyup", onKeyup, true); + }, + options, + ); + dialog.addEventListener( + "cancel", + (event) => { + if (event.target !== dialog) return; + event.preventDefault(); + if (!this._escapeCleanup && this._escapeTimer === null) + this.requestClose("escape"); + }, + options, + ); + dialog.addEventListener( + "close", + (event) => { + if (event.target !== dialog || dialog.open) return; + this._clearPendingClose(); + this.busy = false; + if (this._restoreFocus && this._returnFocusTo?.isConnected) { + // Menu actions may have become hidden while the dialog was open. + const details = this._returnFocusTo.closest("details:not([open])"); + const target = + details?.querySelector("summary") || this._returnFocusTo; + target.focus({ preventScroll: true }); + } + this._returnFocusTo = null; + }, + options, + ); + } + + disconnectedCallback() { + this._listeners?.abort(); + this._clearPendingClose(); + this._returnFocusTo = null; + if (this.dialog?.open) this.dialog.close(); + this.busy = false; + } + + _clearPendingClose() { + this._escapeCleanup?.(); + this._escapeCleanup = null; + clearTimeout(this._escapeTimer); + this._escapeTimer = null; + } + + show({ returnFocusTo, initialFocus } = {}) { + const dialog = this.dialog; + if (!dialog.open) { + this._clearPendingClose(); + this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement; + this._restoreFocus = true; + dialog.showModal(); + } + if (typeof initialFocus === "function") initialFocus(); + else initialFocus?.focus(); + } + + requestClose(source = "cancel") { + if (!this.dialog.open || this.busy) return false; + if (this.beforeClose && this.beforeClose(source) === false) return false; + this.close(); + return true; + } + + close({ restoreFocus = true } = {}) { + this._clearPendingClose(); + this._restoreFocus = restoreFocus; + this.dialog.close(); + } + } + + customElements.define("datasette-modal", DatasetteModal); + window.DatasetteModal = DatasetteModal; +})(); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index ec2d23d8..df2516f9 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -10,277 +10,22 @@ class NavigationSearch extends HTMLElement { this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; this.statusId = `navigation-search-status-${this.instanceId}`; this.titleId = `navigation-search-title-${this.instanceId}`; - this.attachShadow({ mode: "open" }); this.selectedIndex = -1; this.matches = []; this.renderedMatches = []; this.debounceTimer = null; - this.restoreFocusTarget = null; - this.shouldRestoreFocus = true; + } + connectedCallback() { + if (this._initialized) return; + this._initialized = true; this.render(); this.setupEventListeners(); } render() { - this.shadowRoot.innerHTML = ` - - - + this.innerHTML = ` +
      @@ -302,23 +47,22 @@ class NavigationSearch extends HTMLElement { >
      -
      +
      ↑ ↓ Navigate Enter Select Esc Close
      - + `; } setupEventListeners() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const closeButton = this.shadowRoot.querySelector(".close-search"); - const resultsContainer = - this.shadowRoot.querySelector(".results-container"); + const dialog = this.querySelector("dialog"); + const input = this.querySelector(".search-input"); + const closeButton = this.querySelector(".close-search"); + const resultsContainer = this.querySelector(".results-container"); // Global keyboard listener for "/" document.addEventListener("keydown", (e) => { @@ -355,8 +99,6 @@ class NavigationSearch extends HTMLElement { } else if (e.key === "Enter") { e.preventDefault(); this.selectCurrentItem(); - } else if (e.key === "Escape") { - this.closeMenu(); } }); @@ -380,18 +122,6 @@ class NavigationSearch extends HTMLElement { } }); - // Close on backdrop click - dialog.addEventListener("click", (e) => { - if (e.target === dialog) { - this.closeMenu(); - } - }); - - dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this.closeMenu(); - }); - dialog.addEventListener("close", () => { this.onMenuClosed(); }); @@ -432,19 +162,6 @@ class NavigationSearch extends HTMLElement { } } - focusRestoreTarget(trigger) { - if (trigger && typeof trigger.focus === "function") { - return trigger; - } - if ( - document.activeElement && - typeof document.activeElement.focus === "function" - ) { - return document.activeElement; - } - return null; - } - setNavigationTriggersExpanded(expanded) { if (typeof document.querySelectorAll !== "function") { return; @@ -465,8 +182,8 @@ class NavigationSearch extends HTMLElement { } updateComboboxState() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); + const dialog = this.querySelector("dialog"); + const input = this.querySelector(".search-input"); const matches = this.renderedMatches || []; this.setElementAttribute( input, @@ -491,7 +208,7 @@ class NavigationSearch extends HTMLElement { } setStatus(message) { - const status = this.shadowRoot.querySelector(`#${this.statusId}`); + const status = this.querySelector(`#${this.statusId}`); if (status) { status.textContent = message || ""; } @@ -701,7 +418,7 @@ class NavigationSearch extends HTMLElement { section.render(node, { navigationSearch: this, container, - input: this.shadowRoot.querySelector(".search-input"), + input: this.querySelector(".search-input"), }); }); } @@ -740,8 +457,8 @@ class NavigationSearch extends HTMLElement { } renderResults() { - const container = this.shadowRoot.querySelector(".results-container"); - const input = this.shadowRoot.querySelector(".search-input"); + const container = this.querySelector(".results-container"); + const input = this.querySelector(".search-input"); const showStartContent = !input.value.trim(); const jumpSections = showStartContent ? this.jumpSections() : []; const startBlock = showStartContent @@ -853,18 +570,15 @@ class NavigationSearch extends HTMLElement { } } - openMenu(trigger) { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); + openMenu(returnFocusTo) { + const input = this.querySelector(".search-input"); - this.restoreFocusTarget = this.focusRestoreTarget(trigger); - this.shouldRestoreFocus = true; - if (!dialog.open) { - dialog.showModal(); - } + this.querySelector("datasette-modal").show({ + returnFocusTo, + initialFocus: input, + }); this.setNavigationTriggersExpanded(true); input.value = ""; - input.focus(); // Reset state, then populate the default jump list. this.matches = []; @@ -874,29 +588,15 @@ class NavigationSearch extends HTMLElement { } closeMenu(options = {}) { - const dialog = this.shadowRoot.querySelector("dialog"); - this.shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - this.onMenuClosed(); - } + this.querySelector("datasette-modal").close(options); } onMenuClosed() { - const input = this.shadowRoot.querySelector(".search-input"); + const input = this.querySelector(".search-input"); this.setElementAttribute(input, "aria-expanded", "false"); this.removeElementAttribute(input, "aria-activedescendant"); this.setNavigationTriggersExpanded(false); this.setStatus(""); - if ( - this.shouldRestoreFocus && - this.restoreFocusTarget && - typeof this.restoreFocusTarget.focus === "function" - ) { - this.restoreFocusTarget.focus(); - } - this.restoreFocusTarget = null; } escapeHtml(text) { diff --git a/datasette/static/table.js b/datasette/static/table.js index 74a96d8e..6b3e619c 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -157,6 +157,7 @@ function createSetColumnTypeOption(value, name, description, checked) { function setSetColumnTypeDialogBusy(state, isBusy) { state.isBusy = isBusy; + state.modal.busy = isBusy; state.saveButton.disabled = isBusy; state.cancelButton.disabled = isBusy; Array.from( @@ -185,7 +186,8 @@ function ensureSetColumnTypeDialog() { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = SET_COLUMN_TYPE_DIALOG_ID; dialog.className = "set-column-type-dialog"; dialog.setAttribute("aria-labelledby", "set-column-type-title"); @@ -196,16 +198,17 @@ function ensureSetColumnTypeDialog() {

      -
      +
      - - + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); setColumnTypeDialogState = { + modal: modal, dialog: dialog, meta: dialog.querySelector(".modal-meta"), status: dialog.querySelector(".set-column-type-status"), @@ -220,21 +223,7 @@ function ensureSetColumnTypeDialog() { }; setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - if (!setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - if (setColumnTypeDialogState.isBusy) { - ev.preventDefault(); - } + modal.requestClose("cancel"); }); dialog.addEventListener("close", function () { @@ -242,49 +231,52 @@ function ensureSetColumnTypeDialog() { setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); }); - setColumnTypeDialogState.saveButton.addEventListener("click", async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; + setColumnTypeDialogState.saveButton.addEventListener( + "click", + async function () { + var state = setColumnTypeDialogState; + var selected = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + var selectedType = selected ? selected.value : ""; + var currentType = state.currentConfig.current + ? state.currentConfig.current.type + : ""; - if (selectedType === currentType) { - state.dialog.close(); - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); + if (selectedType === currentType) { + state.modal.close(); + return; } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }); + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, true); + + var payload = { + column: state.currentColumn, + column_type: selectedType ? { type: selectedType } : null, + }; + + try { + var response = await fetch(getSetColumnTypeData().path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + var message = (data.errors || ["Request failed"]).join(" "); + throw new Error(message); + } + location.reload(); + } catch (error) { + setSetColumnTypeDialogBusy(state, false); + showSetColumnTypeDialogError(state, error.message || "Request failed"); + } + }, + ); return setColumnTypeDialogState; } @@ -341,9 +333,7 @@ function openSetColumnTypeDialog(th) { state.optionsWrap.appendChild(emptyState); } - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show(); var selectedOption = state.dialog.querySelector( 'input[name="set-column-type-choice"]:checked', ); @@ -367,9 +357,10 @@ function shouldShowShowAllColumns() { function hasMultipleVisibleColumns(manager) { return ( - Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( - (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", - ).length > 1 + Array.from( + document.querySelectorAll(manager.selectors.tableHeaders), + ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1") + .length > 1 ); } @@ -649,10 +640,12 @@ function filterRowNumberFromName(name) { } function nextFilterRowNumber(manager) { - return filterRowsWithControls(manager).reduce((max, row) => { - var column = row.querySelector("select"); - return Math.max(max, filterRowNumberFromName(column && column.name)); - }, 0) + 1; + return ( + filterRowsWithControls(manager).reduce((max, row) => { + var column = row.querySelector("select"); + return Math.max(max, filterRowNumberFromName(column && column.name)); + }, 0) + 1 + ); } function setFilterRowNumber(row, number) { @@ -679,9 +672,11 @@ function updateFilterRowButtons(manager) { if (addButton) { addButton.hidden = index !== rows.length - 1 || !column.value; } - var visibleButtonCount = [removeButton, addButton].filter(function (button) { - return button && !button.hidden; - }).length; + var visibleButtonCount = [removeButton, addButton].filter( + function (button) { + return button && !button.hidden; + }, + ).length; row.classList.toggle( "filter-controls-row-has-buttons", visibleButtonCount > 0, @@ -703,7 +698,9 @@ function cloneFilterRow(row) { clone.querySelector(".filter-op select").name = "_filter_op"; clone.querySelector("input.filter-value").name = "_filter_value"; resetFilterRow(clone); - clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove()); + clone + .querySelectorAll(".filter-row-icon") + .forEach((button) => button.remove()); return clone; } @@ -860,10 +857,45 @@ function openColumnChooser() { }); } +function initCountAll() { + var button = document.querySelector(".count-all"); + if (!button) { + return; + } + button.addEventListener("click", async function () { + var count = document.querySelector(".table-count"); + var error = document.querySelector(".count-error"); + button.disabled = true; + button.textContent = "Counting…"; + error.textContent = ""; + try { + var response = await fetch(button.dataset.countUrl + location.search, { + method: "POST", + headers: { + Accept: "application/json", + }, + }); + var data = await response.json(); + if (!response.ok || !data.ok) { + throw new Error((data.errors || ["Count failed"]).join(" ")); + } + count.textContent = + data.count.toLocaleString("en-US") + + (data.count === 1 ? " row" : " rows"); + button.remove(); + } catch (ex) { + error.textContent = ex.message || "Count failed"; + button.disabled = false; + button.textContent = "count all"; + } + }); +} + // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; + initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index a6123daa..db3c6548 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,8 +1,9 @@ from __future__ import annotations -from dataclasses import dataclass import json -from typing import Any, Iterable +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any from .utils import tilde_encode, urlsafe_components @@ -62,7 +63,6 @@ def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: "description_html": query.description_html, "hide_sql": query.hide_sql, "fragment": query.fragment, - "params": list(query.parameters), "parameters": list(query.parameters), "is_write": query.is_write, "is_private": query.is_private, @@ -84,7 +84,6 @@ def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: return { "queries": [stored_query_to_dict(query) for query in page.queries], "next": page.next, - "has_more": page.has_more, "limit": page.limit, } @@ -388,7 +387,7 @@ async def count_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = "%{}%".format(q) + params["query_search"] = f"%{q}%" if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) @@ -464,7 +463,7 @@ async def list_queries( except ValueError: components = [] if database is None and len(components) == 3: - where_clauses.append(""" + where_clauses.append(f""" ( q.database_name > :cursor_database OR ( @@ -478,12 +477,12 @@ async def list_queries( ) ) ) - """.format(sort_key_sql=sort_key_sql)) + """) params["cursor_database"] = components[0] params["cursor_sort_key"] = components[1] params["cursor_name"] = components[2] elif database is not None and len(components) == 2: - where_clauses.append(""" + where_clauses.append(f""" ( {sort_key_sql} > :cursor_sort_key OR ( @@ -491,7 +490,7 @@ async def list_queries( AND q.name > :cursor_name ) ) - """.format(sort_key_sql=sort_key_sql)) + """) params["cursor_sort_key"] = components[0] params["cursor_name"] = components[1] @@ -504,7 +503,7 @@ async def list_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = "%{}%".format(q) + params["query_search"] = f"%{q}%" if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) diff --git a/datasette/telemetry.py b/datasette/telemetry.py new file mode 100644 index 00000000..2fd5a9dc --- /dev/null +++ b/datasette/telemetry.py @@ -0,0 +1,481 @@ +""" +OpenTelemetry integration for Datasette. + +This uses `opentelemetry-api` only. Providers, exporters and sampling are +configured by whoever runs Datasette, for example `opentelemetry-instrument`. +""" + +import contextvars +import re +import threading +import time +import weakref +from contextlib import contextmanager + +from opentelemetry import context as otel_context_api +from opentelemetry import metrics as otel_metrics +from opentelemetry import trace as otel_trace +from opentelemetry.propagate import extract +from opentelemetry.propagators.textmap import Getter +from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span + +from .telemetry_registry import ( + DB_NAMESPACE, + DB_SYSTEM, + ERROR_TYPE, + HTTP_REQUEST_METHOD, + HTTP_RESPONSE_STATUS_CODE, + INTERNAL_CLIENT, + M_CONNECTIONS_OPEN, + M_OPERATION_DURATION, + M_QUERIES_INTERRUPTED, + M_QUERIES_PENDING, + M_THREADS_LIMIT, + M_THREADS_QUEUE_DEPTH, + M_WRITE_QUEUE_DEPTH, + M_WRITE_QUEUE_WAIT, + OPERATION, + SERVER_ADDRESS, + URL_PATH, + URL_SCHEME, + USER_AGENT_ORIGINAL, +) +from .version import __version__ + +# True while code is executing within a datasette.client request. Defined +# here rather than in app.py to avoid a circular import. +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + +# The semantic conventions version matching the attribute names used here. +# 1.30.0 renamed `db.system` to `db.system.name`, so update this when +# renaming attributes to match a newer version. +SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0" + +tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL) +meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL) + +MAX_SQL_LENGTH = 2048 + + +def sql_attribute(sql: str) -> str: + "Truncate SQL text so it is safe to attach to a span as an attribute." + sql = sql.strip() + if len(sql) <= MAX_SQL_LENGTH: + return sql + return sql[:MAX_SQL_LENGTH] + "…[truncated]" + + +def callback_name(fn) -> str: + """ + The name recorded as `datasette.callback` for a callback-style call. + + Falls back to the type name for callables such as `functools.partial` + that have no `__qualname__`. + """ + return getattr(fn, "__qualname__", type(fn).__name__) + + +def linked_root_span_kwargs(context=None): + """ + Keyword arguments that start a new root span with a ``Link`` back to + the current span. + + Use this for work that can outlive the span that caused it, such as a + background task or a ``block=False`` write. + + Pass ``context`` to link to the span in a previously captured context + instead of the current one. If there is no valid span, no link is added. + + Works with any tracer:: + + with my_tracer.start_as_current_span( + "myplugin.job", **linked_root_span_kwargs() + ): + ... + """ + cause = get_current_span(context).get_span_context() + links = [Link(cause)] if cause.is_valid else [] + return {"context": otel_context_api.Context(), "links": links} + + +# Keywords that can be recorded as db.operation.name. SQL can be supplied by +# users, so an allowlist keeps the number of distinct values small. +DB_OPERATION_ALLOWLIST = frozenset( + { + "SELECT", + "INSERT", + "UPDATE", + "DELETE", + "CREATE", + "DROP", + "ALTER", + "PRAGMA", + "EXPLAIN", + "REPLACE", + "VACUUM", + "ANALYZE", + "WITH", + } +) + +_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)") + + +def sql_operation_name(sql: str) -> str | None: + """ + The statement's leading keyword if it is in the allowlist, else None. + + Statements that start with a comment or "(" return None. Statements + starting with a CTE return `WITH`. Only call this for a single statement. + """ + match = _LEADING_KEYWORD.match(sql) + if not match: + return None + keyword = match.group(1).upper() + if keyword in DB_OPERATION_ALLOWLIST: + return keyword + return None + + +# --- The HTTP request span ------------------------------------------------ + + +class _ScopeHeadersGetter(Getter): + "Read W3C trace context from an ASGI scope's headers." + + def get(self, carrier, key): + wanted = key.lower().encode("latin-1") + values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted] + return values or None + + def keys(self, carrier): + return [k.decode("latin-1") for k, _ in carrier] + + +_HEADERS_GETTER = _ScopeHeadersGetter() + + +# Methods defined by RFC 9110 plus PATCH (RFC 5789). Anything else is +# recorded as `_OTHER`, as recommended by semantic conventions. +_KNOWN_METHODS = frozenset( + {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"} +) + + +def clamp_http_method(method): + "The request method if it is one we recognise, else ``_OTHER``." + method = (method or "").upper() + return method if method in _KNOWN_METHODS else "_OTHER" + + +def _first_header(headers, name): + "The first value of a header, decoded, or None." + for key, value in headers: + if key.lower() == name: + return value.decode("latin-1") + return None + + +def _url_path(scope): + """ + The request path, with any query string removed. + + Prefers `raw_path`, which preserves encoded slashes in database and + table names. Some clients include the query string in `raw_path`, so + that is stripped as well. + """ + raw_path = scope.get("raw_path") + if raw_path: + if isinstance(raw_path, bytes): + raw_path = raw_path.decode("latin-1") + return raw_path.split("?", 1)[0] + return scope.get("path", "") + + +# The request span is passed to the router in the ASGI scope, because a +# plugin's asgi_wrapper() middleware may have made its own span current. +# Absent if the span is not recording. +REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span" + + +def request_span(scope): + """ + The recording request span for an ASGI scope, or None. + + Falls back to the current span, for when Datasette is running under + other instrumentation. + """ + span = scope.get(REQUEST_SPAN_SCOPE_KEY) + if span is None: + span = otel_trace.get_current_span() + return span if span.is_recording() else None + + +class TelemetryMiddleware: + """ + One `SpanKind.SERVER` span per HTTP request. + + The span ends after the full response, including any streamed body, + has been sent. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + # Pass lifespan and websocket scopes straight through + if scope["type"] != "http": + await self.app(scope, receive, send) + return + headers = scope.get("headers") or [] + # Uses the global propagator, configured with OTEL_PROPAGATORS + context = extract(headers, getter=_HEADERS_GETTER) + method = clamp_http_method(scope.get("method", "")) + # Renamed to include the route once routing has happened + with tracer.start_as_current_span( + method, context=context, kind=SpanKind.SERVER + ) as span: + if not span.is_recording(): + # No provider installed, or the trace was not sampled + await self.app(scope, receive, send) + return + span.set_attribute(HTTP_REQUEST_METHOD, method) + span.set_attribute(URL_PATH, _url_path(scope)) + scheme = scope.get("scheme") + if scheme: + span.set_attribute(URL_SCHEME, scheme) + host = _first_header(headers, b"host") + if host: + span.set_attribute(SERVER_ADDRESS, host) + user_agent = _first_header(headers, b"user-agent") + if user_agent: + span.set_attribute(USER_AGENT_ORIGINAL, user_agent) + if _in_datasette_client.get(): + span.set_attribute(INTERNAL_CLIENT, True) + + scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span}) + + # Some responses are sent without a Response object, so the + # status is captured by wrapping send() + status_holder = {} + + async def wrapped_send(message): + if ( + message["type"] == "http.response.start" + and "status" not in status_holder + ): + status_holder["status"] = message["status"] + await send(message) + + escaped = False + try: + await self.app(scope, receive, wrapped_send) + except BaseException as exception: + # Includes asyncio.CancelledError when a client disconnects + escaped = True + span.set_attribute(ERROR_TYPE, type(exception).__name__) + span.set_status(Status(StatusCode.ERROR, str(exception))) + raise + finally: + status = status_holder.get("status") + if status is not None: + span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status) + # 4xx responses are not errors for a server span. If an + # exception escaped, keep its class name as error.type. + if status >= 500 and not escaped: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute(ERROR_TYPE, str(status)) + + +# --- Metrics -------------------------------------------------------------- + + +def _duration_attributes(database_name, operation): + return { + DB_SYSTEM: "sqlite", + DB_NAMESPACE: database_name, + OPERATION: operation, + } + + +# Instruments use plain text descriptions. The registry entries have longer +# reStructuredText descriptions for the documentation. + +sql_operation_duration = meter.create_histogram( + M_OPERATION_DURATION, + unit=M_OPERATION_DURATION.unit, + description="Duration of a SQL operation issued by Datasette", + explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets, +) + +write_queue_wait = meter.create_histogram( + M_WRITE_QUEUE_WAIT, + unit=M_WRITE_QUEUE_WAIT.unit, + description=( + "Time a write spent queued behind the single write thread for its database" + ), + explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets, +) + +queries_interrupted = meter.create_counter( + M_QUERIES_INTERRUPTED, + unit=M_QUERIES_INTERRUPTED.unit, + description="Queries cancelled for exceeding sql_time_limit_ms", +) + + +@contextmanager +def record_operation_duration(database_name, operation): + """ + Record `db.client.operation.duration` for one SQL operation. + + Sets `error.type` to the exception class on failure. For a `block=False` + write this measures the time taken to enqueue the write. + """ + attributes = _duration_attributes(database_name, operation) + started = time.perf_counter() + try: + yield + except BaseException as exception: + attributes[ERROR_TYPE] = type(exception).__qualname__ + raise + finally: + sql_operation_duration.record(time.perf_counter() - started, attributes) + + +def record_write_queue_wait(database_name, waited_ns): + write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name}) + + +def record_query_interrupted(database_name): + queries_interrupted.add(1, {DB_NAMESPACE: database_name}) + + +# Live Datasette instances reported by the gauges below. The lock is needed +# because gauge callbacks run on the SDK's collection thread. +# +# The pool gauges do not identify which instance they came from, so they +# are only meaningful for a process running a single Datasette instance. +_live_datasettes = weakref.WeakSet() +_live_datasettes_lock = threading.Lock() + + +def register_datasette(ds): + "Start reporting pool/queue gauges for this Datasette instance." + with _live_datasettes_lock: + _live_datasettes.add(ds) + + +def unregister_datasette(ds): + "Stop reporting gauges for an instance that has been closed." + with _live_datasettes_lock: + _live_datasettes.discard(ds) + + +def _live_instances(): + with _live_datasettes_lock: + return list(_live_datasettes) + + +def _databases_of(ds): + "Every Database attached to an instance, including the internal database." + databases = list(ds.databases.values()) + internal = getattr(ds, "_internal_database", None) + if internal is not None: + databases.append(internal) + return databases + + +def observe_sql_thread_limit(options=None): + "Size of the shared read-query thread pool (the num_sql_threads setting)." + for ds in _live_instances(): + if ds.executor is None: + # num_sql_threads=0 - queries run on the event loop, no pool. + continue + yield otel_metrics.Observation(ds.setting("num_sql_threads"), {}) + + +def observe_sql_thread_queue_depth(options=None): + """ + Read queries waiting for a free thread in the shared pool. + + `_work_queue` is a private attribute of ThreadPoolExecutor, so this + reports nothing if it is missing. + """ + for ds in _live_instances(): + if ds.executor is None: + continue + work_queue = getattr(ds.executor, "_work_queue", None) + if work_queue is None: + continue + yield otel_metrics.Observation(work_queue.qsize(), {}) + + +def observe_pending_queries(options=None): + """ + Read queries submitted to the pool and not yet finished, per database. + + Reads `len()` without `_pending_execute_futures_lock` to avoid blocking + queries. + """ + for ds in _live_instances(): + for db in _databases_of(ds): + yield otel_metrics.Observation( + len(db._pending_execute_futures), {DB_NAMESPACE: db.name} + ) + + +def observe_write_queue_depth(options=None): + "Writes queued behind the single write thread, per database." + for ds in _live_instances(): + for db in _databases_of(ds): + write_queue = db._write_queue + if write_queue is None: + # No write has ever been queued for this database. + continue + yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name}) + + +def observe_open_connections(options=None): + "Open SQLite connections tracked for closing, per database." + for ds in _live_instances(): + for db in _databases_of(ds): + yield otel_metrics.Observation( + len(db._all_connections), {DB_NAMESPACE: db.name} + ) + + +sql_thread_limit_gauge = meter.create_observable_gauge( + M_THREADS_LIMIT, + callbacks=[observe_sql_thread_limit], + unit=M_THREADS_LIMIT.unit, + description="Maximum concurrent read queries (the num_sql_threads setting)", +) + +sql_thread_queue_depth_gauge = meter.create_observable_gauge( + M_THREADS_QUEUE_DEPTH, + callbacks=[observe_sql_thread_queue_depth], + unit=M_THREADS_QUEUE_DEPTH.unit, + description="Read queries waiting for a free thread in the shared SQL pool", +) + +pending_queries_gauge = meter.create_observable_gauge( + M_QUERIES_PENDING, + callbacks=[observe_pending_queries], + unit=M_QUERIES_PENDING.unit, + description="Read queries submitted to the pool and not yet complete", +) + +write_queue_depth_gauge = meter.create_observable_gauge( + M_WRITE_QUEUE_DEPTH, + callbacks=[observe_write_queue_depth], + unit=M_WRITE_QUEUE_DEPTH.unit, + description="Writes queued behind a database's single write thread", +) + +open_connections_gauge = meter.create_observable_gauge( + M_CONNECTIONS_OPEN, + callbacks=[observe_open_connections], + unit=M_CONNECTIONS_OPEN.unit, + description="Open SQLite connections tracked for closing", +) diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py new file mode 100644 index 00000000..63d5b89b --- /dev/null +++ b/datasette/telemetry_registry.py @@ -0,0 +1,502 @@ +""" +Every span, metric and attribute that Datasette emits. + +These entries are used by the instrumentation code, by `docs/telemetry_doc.py` +to generate the documentation, and by `tests/test_telemetry_registry.py` to +check that the emitted telemetry matches the registry. +""" + +from opentelemetry.trace import SpanKind + + +class Attribute(str): + """ + A span attribute key, carrying its own documentation. + + Subclasses `str` so it can be handed straight to `set_attribute()`. + + Part of Datasette's public plugin API - plugins declare their own + telemetry registries with these classes. See the "Telemetry for plugin + authors" documentation. + """ + + __slots__ = ("description", "optional", "values") + + def __new__(cls, name, description, optional=False, values=None): + self = super().__new__(cls, name) + self.description = description + self.optional = optional + # The allowed values for this attribute, or None to allow any value + self.values = frozenset(values) if values is not None else None + return self + + def __reduce__(self): + # Copies and pickles become a plain str, since __new__ requires the + # extra arguments. ConsoleMetricExporter deepcopies attribute keys. + return (str, (str(self),)) + + def __repr__(self): + return f"Attribute({str(self)!r})" + + +class SpanName(str): + """A span name, carrying its documentation and the attributes it may set. + + Part of Datasette's public plugin API, like `Attribute`. + """ + + __slots__ = ("attributes", "description", "dynamic", "kind", "prefix") + + def __new__( + cls, + name, + description, + attributes=(), + prefix=False, + dynamic=False, + kind=SpanKind.INTERNAL, + ): + self = super().__new__(cls, name) + self.description = description + self.attributes = tuple(attributes) + # Match emitted names that start with this prefix, for names with a + # variable suffix such as SpanName("chat ", ..., prefix=True) + self.prefix = prefix + # The emitted name is built at runtime, so `span_for()` matches it by + # span kind. The entry's string is a template for the documentation. + self.dynamic = dynamic + self.kind = kind + return self + + def __reduce__(self): + # See Attribute.__reduce__. + return (str, (str(self),)) + + def __repr__(self): + return f"SpanName({str(self)!r})" + + +class MetricName(str): + "A metric name, carrying its instrument kind, unit and attributes." + + __slots__ = ("attributes", "buckets", "description", "kind", "unit") + + def __new__(cls, name, kind, unit, description, attributes=(), buckets=None): + self = super().__new__(cls, name) + self.kind = kind + self.unit = unit + self.description = description + self.attributes = tuple(attributes) + # Explicit bucket boundaries, for histograms only + self.buckets = tuple(buckets) if buckets is not None else None + return self + + def __reduce__(self): + # See Attribute.__reduce__. + return (str, (str(self),)) + + def __repr__(self): + return f"MetricName({str(self)!r})" + + +COUNTER = "Counter" +UPDOWN_COUNTER = "UpDownCounter" +HISTOGRAM = "Histogram" +GAUGE = "Observable gauge" + + +# --- Attributes ----------------------------------------------------------- + +HTTP_REQUEST_METHOD = Attribute( + "http.request.method", + "The HTTP request method. Methods outside the nine defined by RFC 9110 " + "and RFC 5789 are recorded as ``_OTHER``.", +) +HTTP_RESPONSE_STATUS_CODE = Attribute( + "http.response.status_code", + "The HTTP response status code. Omitted if no response was started.", + optional=True, +) +HTTP_ROUTE = Attribute( + "http.route", + "The regular expression for the matched route, for example " + "``/(?P[^\\/\\.]+)/(?P
      [^\\/\\.]+)(\\.(?P\\w+))?$`` " + "for a table page. Use this attribute to group requests by route. " + "Omitted when no route matches.", + optional=True, +) +URL_PATH = Attribute( + "url.path", + "The URL path, excluding the query string.", +) +URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.") +SERVER_ADDRESS = Attribute( + "server.address", + "The ``Host`` header, including any ``:port`` suffix. This value is " + "supplied by the client.", + optional=True, +) +USER_AGENT_ORIGINAL = Attribute( + "user_agent.original", + "The ``User-Agent`` header, verbatim. Omitted if the client sent none.", + optional=True, +) +INTERNAL_CLIENT = Attribute( + "datasette.internal_client", + "``True`` for requests made through ``datasette.client``. Calls made " + "inside another request produce a nested ``SERVER`` span. Filter on " + "this attribute to exclude internal requests from request counts. " + "Omitted for requests received over the network.", + optional=True, +) +ERROR_TYPE = Attribute( + "error.type", + "The exception class name for a failed operation. On HTTP spans, also " + "set to the status code as a string for 5xx responses. A 4xx response " + "alone does not set this attribute or an error status.", + optional=True, +) + +DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") +DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") +OPERATION = Attribute( + "datasette.operation", + "Whether the operation was a read or a write.", + values={"read", "write"}, +) +DB_QUERY_TEXT = Attribute( + "db.query.text", + "The SQL, truncated to 2048 characters. Bound parameter values are not " + "recorded. For callback methods, ``datasette.callback`` is recorded instead.", + optional=True, +) +CALLBACK = Attribute( + "datasette.callback", + "The qualified name of the Python callable passed to ``execute_fn()``, " + "``execute_write_fn()`` or ``execute_isolated_fn()``, for example " + "``TableInsertView.post..insert_or_upsert_rows``. Set instead of " + "``db.query.text``. Lambdas appear as ````; use a named function " + "for a more descriptive span.", + optional=True, +) +DB_OPERATION_NAME = Attribute( + "db.operation.name", + "The statement's leading keyword, such as ``SELECT``, ``INSERT`` or " + "``CREATE``, if it matches the supported allowlist. Statements beginning " + "with a common table expression report ``WITH``. Omitted for unrecognized " + "keywords and ``execute_write_script()``.", + optional=True, +) +PARAM_COUNT = Attribute( + "datasette.param_count", + "Number of bound parameters. Recorded instead of the values themselves.", + optional=True, +) +PARAM_SETS = Attribute( + "datasette.param_sets", + "Number of parameter sets consumed by ``execute_write_many()``. " + "The parameter values are not recorded.", + optional=True, +) +TIME_LIMIT_MS = Attribute( + "datasette.time_limit_ms", + "Time limit applied to the read query, in milliseconds: " + ":ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.", + optional=True, +) +ROWS_RETURNED = Attribute( + "datasette.rows_returned", + "Number of rows returned by a successful read query.", + optional=True, +) +TRUNCATED = Attribute( + "datasette.truncated", + "True if the result was cut short by :ref:`setting_max_returned_rows`.", + optional=True, +) +INTERRUPTED = Attribute( + "datasette.interrupted", + "True if the query exceeded its time limit. The span status is set to " + "``ERROR`` unless the caller used a ``custom_time_limit`` shorter than " + ":ref:`setting_sql_time_limit_ms`, in which case the status is left unset.", + optional=True, +) +SQL_ERROR_SUPPRESSED = Attribute( + "datasette.sql_error_suppressed", + "True for a non-timeout SQL error with ``log_sql_errors=False``. The " + "exception is still raised, but the span status is left unset.", + optional=True, +) +EXECUTESCRIPT = Attribute( + "datasette.executescript", + "True for ``execute_write_script()``, which runs multiple statements.", + optional=True, +) +EXECUTEMANY = Attribute( + "datasette.executemany", + "True for ``execute_write_many()``, which runs one statement against many " + "parameter sets.", + optional=True, +) +ISOLATED_CONNECTION = Attribute( + "datasette.isolated_connection", + "True if the write ran on its own connection rather than the shared write " + "connection.", +) +TRANSACTION = Attribute( + "datasette.transaction", + "False for statements such as ``VACUUM`` that cannot run inside a transaction.", +) + + +# --- Spans ---------------------------------------------------------------- + +HTTP_REQUEST = SpanName( + "{http.request.method} {http.route}", + "One span per HTTP request, containing spans from plugin middleware and " + "database operations. Named for the HTTP method and matched route, or " + "just the method if no route matches. Incoming ``traceparent`` headers " + "are extracted using the global propagator to continue the caller's " + "trace. Incoming ``baggage`` is not propagated into plugin or downstream " + "context in this release. Set ``OTEL_PROPAGATORS=none`` to disable " + "extraction. For public instances, strip trace context headers at your " + "proxy if callers should not supply trace context.", + ( + HTTP_REQUEST_METHOD, + HTTP_ROUTE, + URL_PATH, + URL_SCHEME, + SERVER_ADDRESS, + USER_AGENT_ORIGINAL, + HTTP_RESPONSE_STATUS_CODE, + ERROR_TYPE, + INTERNAL_CLIENT, + ), + dynamic=True, + kind=SpanKind.SERVER, +) + +DB_QUERY = SpanName( + "db.query", + "A SQL operation, including time spent queued for a worker thread. For " + "``block=False`` writes, the span ends after the write is queued. " + "Callback methods record ``datasette.callback`` in place of ``db.query.text``.", + ( + DB_SYSTEM, + DB_NAMESPACE, + DB_QUERY_TEXT, + CALLBACK, + DB_OPERATION_NAME, + PARAM_COUNT, + PARAM_SETS, + TIME_LIMIT_MS, + ROWS_RETURNED, + TRUNCATED, + INTERRUPTED, + SQL_ERROR_SUPPRESSED, + EXECUTESCRIPT, + EXECUTEMANY, + ), + kind=SpanKind.CLIENT, +) + +DB_QUERY_EXECUTE = SpanName( + "db.query.execute", + "The read executing inside a SQL worker thread. Child of ``db.query``; the " + "gap between the two is time spent waiting for a thread.", +) + +DB_WRITE_QUEUE_WAIT = SpanName( + "db.write.queue_wait", + "Time a write spent waiting in its database's write queue. For " + "``block=True``, this is a child of ``db.query``. For ``block=False``, " + "it is a root span linked to the span that queued the write, since the " + "write can outlive that request.", +) + +DB_WRITE_EXECUTE = SpanName( + "db.write.execute", + "The write executing on the write thread. For ``block=True``, this is " + "a child of ``db.query``. For ``block=False``, it is a root span linked " + "to the span that queued the write.", + (ISOLATED_CONNECTION, TRANSACTION), +) + +STARTUP = SpanName( + "datasette.startup", + "Startup work performed by ``invoke_startup()``, including registration " + "hooks, schema catalog updates, saved queries, column type configuration " + "and the ``startup`` hook. Runs during instance startup, either before " + "serving requests or as part of the first request.", +) + +SPANS = ( + HTTP_REQUEST, + DB_QUERY, + DB_QUERY_EXECUTE, + DB_WRITE_QUEUE_WAIT, + DB_WRITE_EXECUTE, + STARTUP, +) + + +def span_for(emitted_name, kind=None, spans=None): + """ + Resolve an emitted span name to its registry entry, or None. + + Exact matches take precedence over `prefix=True` entries, which take + precedence over `dynamic=True` entries matched by `kind`. + + `spans` defaults to Datasette's own registry. + """ + if spans is None: + spans = SPANS + for span in spans: + if span.dynamic: + continue + if emitted_name == span: + return span + for span in spans: + if span.prefix and emitted_name.startswith(span): + return span + if kind is not None: + for span in spans: + if span.dynamic and span.kind == kind: + return span + return None + + +def metric_for(emitted_name, metrics=None): + """ + Resolve an emitted metric name to its registry entry, or None. + + `metrics` defaults to Datasette's own registry. + """ + if metrics is None: + metrics = METRICS + for metric in metrics: + if emitted_name == metric: + return metric + return None + + +def attribute_allowed(entry, emitted_key): + """ + Whether `emitted_key` is a registered attribute of `entry`. + + `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`. + """ + if entry is None: + return False + return emitted_key in entry.attributes + + +def attribute_value_allowed(entry, emitted_key, value): + """ + Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName` + or a `MetricName`). + + Any value is allowed if the attribute does not declare `values=`. + """ + if entry is None: + return False + for attribute in entry.attributes: + if attribute == emitted_key: + return attribute.values is None or value in attribute.values + return False + + +# --- Metrics -------------------------------------------------------------- + +# Bucket boundaries in seconds for every duration histogram. OpenTelemetry's +# defaults are designed for milliseconds and would put almost every SQLite +# query in the first bucket. These are the semantic conventions' recommended +# boundaries for db.client.operation.duration, plus 0.0001 and 0.0005 for +# fast in-process SQLite queries. +DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10) + +M_OPERATION_DURATION = MetricName( + "db.client.operation.duration", + HISTOGRAM, + "s", + "Duration of a SQL operation, including callback-based calls such as " + "``execute_fn()``. For ``block=False`` writes, measures enqueue time.", + (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE), + buckets=DURATION_BUCKETS, +) + +M_WRITE_QUEUE_WAIT = MetricName( + "datasette.write.queue_wait", + HISTOGRAM, + "s", + "Time each write waited in its database's write queue.", + (DB_NAMESPACE,), + buckets=DURATION_BUCKETS, +) + +M_QUERIES_INTERRUPTED = MetricName( + "datasette.sql.queries.interrupted", + COUNTER, + "{query}", + "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A " + "rising rate can indicate that queries need optimization or a higher " + "time limit. Caller-selected timeouts shorter than this limit, such as " + "those used for facet suggestion, are excluded.", + (DB_NAMESPACE,), +) + +M_THREADS_LIMIT = MetricName( + "datasette.sql.threads.limit", + GAUGE, + "{thread}", + "Maximum concurrent read queries, configured by " + ":ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` " + "is ``0``.", +) + +M_THREADS_QUEUE_DEPTH = MetricName( + "datasette.sql.threads.queue_depth", + GAUGE, + "{query}", + "Read queries waiting for a free SQL thread. Sustained values above " + "zero indicate a saturated read pool.", +) + +M_QUERIES_PENDING = MetricName( + "datasette.sql.queries.pending", + GAUGE, + "{query}", + "Read queries submitted to the pool and not yet complete. Sum across " + "databases and compare with ``datasette.sql.threads.limit`` to assess " + "pool usage.", + (DB_NAMESPACE,), +) + +M_WRITE_QUEUE_DEPTH = MetricName( + "datasette.write.queue_depth", + GAUGE, + "{write}", + "Writes waiting for a database's single write thread. Increasing " + "``num_sql_threads`` does not increase write concurrency. Not reported for " + "databases that have never been written to.", + (DB_NAMESPACE,), +) + +M_CONNECTIONS_OPEN = MetricName( + "datasette.connections.open", + GAUGE, + "{connection}", + "Open SQLite connections managed by Datasette.", + (DB_NAMESPACE,), +) + +METRICS = ( + M_OPERATION_DURATION, + M_WRITE_QUEUE_WAIT, + M_QUERIES_INTERRUPTED, + M_THREADS_LIMIT, + M_THREADS_QUEUE_DEPTH, + M_QUERIES_PENDING, + M_WRITE_QUEUE_DEPTH, + M_CONNECTIONS_OPEN, +) diff --git a/datasette/telemetry_testing.py b/datasette/telemetry_testing.py new file mode 100644 index 00000000..77a4431b --- /dev/null +++ b/datasette/telemetry_testing.py @@ -0,0 +1,427 @@ +""" +Pytest helpers for testing OpenTelemetry instrumentation - Datasette's own +and any plugin's. Part of Datasette's public plugin API; see the "Telemetry +for plugin authors" documentation. + +Usage from a plugin's ``conftest.py``:: + + from datasette.telemetry_testing import ( # noqa: F401 + MetricsCollector, + otel_metrics, + otel_meter_provider, + otel_provider, + otel_spans, + ) + +Tests can then use the ``otel_spans`` and ``otel_metrics`` fixtures. The +OpenTelemetry SDK is imported lazily, and the fixtures skip if it is not +installed. +""" + +import subprocess +import sys + +import pytest + +from .telemetry_registry import ( + attribute_allowed, + attribute_value_allowed, + metric_for, + span_for, +) + +_span_exporter = None +_metric_reader = None + + +def install_span_exporter(): + """ + Install a TracerProvider + InMemorySpanExporter once per process and + return the exporter, or None when the SDK is not installed. + + Uses `SimpleSpanProcessor` so spans are exported as soon as they end. + """ + global _span_exporter + if _span_exporter is not None: + return _span_exporter + try: + from opentelemetry import trace as otel_trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + except ImportError: + return None + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel_trace.set_tracer_provider(provider) + # set_tracer_provider() is ignored if a provider was already installed, + # in which case the fixtures skip + if otel_trace.get_tracer_provider() is not provider: + return None + _span_exporter = exporter + return exporter + + +def install_metric_reader(): + """ + Install a MeterProvider + InMemoryMetricReader once per process and + return the reader, or None when the SDK is not installed. + + Uses delta temporality for counters and histograms, so each collection + only reports measurements since the previous one. + """ + global _metric_reader + if _metric_reader is not None: + return _metric_reader + try: + from opentelemetry import metrics as otel_metrics_api + from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + InMemoryMetricReader, + ) + except ImportError: + return None + reader = InMemoryMetricReader( + preferred_temporality={ + Counter: AggregationTemporality.DELTA, + Histogram: AggregationTemporality.DELTA, + } + ) + provider = MeterProvider(metric_readers=[reader]) + otel_metrics_api.set_meter_provider(provider) + if otel_metrics_api.get_meter_provider() is not provider: + return None + _metric_reader = reader + return reader + + +@pytest.fixture(scope="session", autouse=True) +def otel_provider(): + "Install the span exporter once per test session, before any spans are created." + install_span_exporter() + + +@pytest.fixture(scope="session", autouse=True) +def otel_meter_provider(): + "Install the metric reader once per test session." + install_metric_reader() + + +@pytest.fixture(autouse=True) +def otel_reset(): + "Clear recorded spans and drain collected metrics after every test." + yield + if _span_exporter is not None: + _span_exporter.clear() + if _metric_reader is not None: + _metric_reader.get_metrics_data() + + +@pytest.fixture +def otel_spans(): + """ + The in-memory span exporter, cleared before the test. Call + `.get_finished_spans()` to retrieve spans. + """ + pytest.importorskip("opentelemetry.sdk") + exporter = install_span_exporter() + if exporter is None: + pytest.skip("OpenTelemetry SDK provider was not installed") + exporter.clear() + yield exporter + + +class MetricsCollector: + """ + Wraps an `InMemoryMetricReader`. + + `collect()` runs a collection cycle and stores a snapshot, which + `points()` and `point()` then query. + """ + + def __init__(self, reader): + self.reader = reader + self.snapshot = {} + # (instrumentation scope name, sdk Metric) pairs from the last collect() + self.collected = [] + + def collect(self): + self.snapshot = {} + self.collected = [] + data = self.reader.get_metrics_data() + if data is None: + return self.snapshot + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + scope_name = scope_metrics.scope.name if scope_metrics.scope else None + for metric in scope_metrics.metrics: + self.snapshot.setdefault(metric.name, []).extend( + metric.data.data_points + ) + self.collected.append((scope_name, metric)) + return self.snapshot + + def points(self, name, attributes=None): + "Data points for `name` whose attributes are a superset of `attributes`." + found = [] + for point in self.snapshot.get(name, []): + point_attributes = dict(point.attributes or {}) + if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()): + found.append(point) + return found + + def point(self, name, attributes=None): + "The single matching data point, asserting there is exactly one." + found = self.points(name, attributes) + assert len(found) == 1, ( + f"expected exactly one {name} point matching {attributes}, " + f"got {len(found)}: {found}" + ) + return found[0] + + +@pytest.fixture +def otel_metrics(): + "A `MetricsCollector`, drained before the test so counts start from zero." + pytest.importorskip("opentelemetry.sdk") + reader = install_metric_reader() + if reader is None: + pytest.skip("OpenTelemetry SDK meter provider was not installed") + reader.get_metrics_data() + yield MetricsCollector(reader) + + +def _scoped(finished_spans, scope_name): + if scope_name is None: + return list(finished_spans) + return [ + span + for span in finished_spans + if span.instrumentation_scope and span.instrumentation_scope.name == scope_name + ] + + +def assert_spans_conform(registry_spans, finished_spans, scope_name=None): + """ + Assert every finished span is registered in `registry_spans`, sets only + registered attributes and uses allowed attribute values. + + Pass `scope_name` to only check spans from that instrumentation scope. + """ + problems = [] + for span in _scoped(finished_spans, scope_name): + entry = span_for(str(span.name), kind=span.kind, spans=registry_spans) + if entry is None: + problems.append(f"unregistered span: {span.name!r}") + continue + for key, value in (span.attributes or {}).items(): + if not attribute_allowed(entry, str(key)): + problems.append(f"{span.name}: unregistered attribute {key!r}") + elif not attribute_value_allowed(entry, str(key), value): + problems.append( + f"{span.name}: {key}={value!r} not in the declared enum" + ) + assert not problems, "\n".join(problems) + + +def assert_spans_covered(registry_spans, finished_spans, scope_name=None): + """ + Assert every entry in `registry_spans` was emitted at least once, with + each of its attributes that is not `optional=True`. + """ + spans = _scoped(finished_spans, scope_name) + seen_attributes = {} + for span in spans: + entry = span_for(str(span.name), kind=span.kind, spans=registry_spans) + if entry is not None: + seen = seen_attributes.setdefault(str(entry), set()) + seen.update(str(key) for key in (span.attributes or {})) + problems = [] + for entry in registry_spans: + if str(entry) not in seen_attributes: + problems.append(f"registered span never emitted: {entry!r}") + continue + required = { + str(attribute) for attribute in entry.attributes if not attribute.optional + } + missing = required - seen_attributes[str(entry)] + if missing: + problems.append( + f"{entry}: registered attributes never emitted: {sorted(missing)}" + ) + assert not problems, "\n".join(problems) + + +# Registry instrument kinds mapped to the SDK data type collected for them. +# Both counter kinds collect as Sum, distinguished by is_monotonic. +_KIND_TO_DATA_TYPE = { + "Counter": "Sum", + "UpDownCounter": "Sum", + "Histogram": "Histogram", + "Observable gauge": "Gauge", +} +_KIND_IS_MONOTONIC = {"Counter": True, "UpDownCounter": False} + + +def _scoped_metrics(collector, scope_name): + for scope, metric in collector.collected: + if scope_name is None or scope == scope_name: + yield metric + + +def assert_metrics_conform(registry_metrics, collector, scope_name=None): + """ + Assert every metric in the collector's last `collect()` is registered in + `registry_metrics` with a matching instrument kind and unit, sets only + registered attributes and uses allowed attribute values. + + Pass `scope_name` to only check metrics from that instrumentation scope. + """ + problems = set() + for metric in _scoped_metrics(collector, scope_name): + entry = metric_for(metric.name, metrics=registry_metrics) + if entry is None: + problems.add(f"unregistered metric: {metric.name!r}") + continue + expected_data_type = _KIND_TO_DATA_TYPE.get(entry.kind) + actual_data_type = type(metric.data).__name__ + if expected_data_type is not None and actual_data_type != expected_data_type: + problems.add( + f"{metric.name}: registry declares {entry.kind}, " + f"SDK collected {actual_data_type}" + ) + expected_monotonic = _KIND_IS_MONOTONIC.get(entry.kind) + actual_monotonic = getattr(metric.data, "is_monotonic", None) + if ( + expected_monotonic is not None + and actual_monotonic is not None + and actual_monotonic != expected_monotonic + ): + problems.add( + f"{metric.name}: registry declares {entry.kind}, but the " + f"collected Sum is_monotonic={actual_monotonic}" + ) + if (metric.unit or "") != (entry.unit or ""): + problems.add( + f"{metric.name}: instrument unit {metric.unit!r} != " + f"registry unit {entry.unit!r}" + ) + for point in metric.data.data_points: + for key, value in dict(point.attributes or {}).items(): + if not attribute_allowed(entry, str(key)): + problems.add(f"{metric.name}: unregistered attribute {key!r}") + elif not attribute_value_allowed(entry, str(key), value): + problems.add( + f"{metric.name}: {key}={value!r} not in the declared enum" + ) + assert not problems, "\n".join(sorted(problems)) + + +def assert_metrics_covered(registry_metrics, collector, scope_name=None): + """ + Assert every entry in `registry_metrics` was collected at least once, + with each of its attributes that is not `optional=True`. + + Call `collect()` once after the workload and before this check. + """ + seen_attributes = {} + for metric in _scoped_metrics(collector, scope_name): + entry = metric_for(metric.name, metrics=registry_metrics) + if entry is None: + continue + seen = seen_attributes.setdefault(str(entry), set()) + for point in metric.data.data_points: + seen.update(str(key) for key in dict(point.attributes or {})) + problems = [] + for entry in registry_metrics: + if str(entry) not in seen_attributes: + problems.append(f"registered metric never collected: {entry!r}") + continue + required = { + str(attribute) for attribute in entry.attributes if not attribute.optional + } + missing = required - seen_attributes[str(entry)] + if missing: + problems.append( + f"{entry}: registered attributes never collected: {sorted(missing)}" + ) + assert not problems, "\n".join(problems) + + +def assert_no_forbidden_values( + forbidden, finished_spans=None, collector=None, scope_name=None +): + """ + Assert that none of the `forbidden` strings appear anywhere in the + emitted telemetry: span names, span attribute values, span event names + and attributes, span status descriptions, or metric point attributes. + + Use fake private values such as tokens or email addresses in your test + workload, then check that they were not recorded: + + FORBIDDEN = {"secret-token-123", "alice@example.com"} + run_workload_using_those_values() + assert_no_forbidden_values( + FORBIDDEN, + finished_spans=otel_spans.get_finished_spans(), + collector=otel_metrics, + ) + + Matches substrings of each value's string form. Empty strings in + `forbidden` are ignored. Leave `scope_name` unset to also check + Datasette's own telemetry. + """ + needles = [needle for needle in forbidden if needle] + leaks = set() + + def check(value, where): + text = str(value) + for needle in needles: + if needle in text: + leaks.add(f"{where} contains {needle!r}") + + if finished_spans is not None: + for span in _scoped(finished_spans, scope_name): + check(span.name, f"span name {str(span.name)!r}") + for key, value in (span.attributes or {}).items(): + check(value, f"{span.name} attribute {key}") + for event in span.events or (): + check(event.name, f"{span.name} event name") + for key, value in (event.attributes or {}).items(): + check(value, f"{span.name} event {event.name} attribute {key}") + if span.status is not None and span.status.description: + check(span.status.description, f"{span.name} status description") + if collector is not None: + for metric in _scoped_metrics(collector, scope_name): + for point in metric.data.data_points: + for key, value in dict(point.attributes or {}).items(): + check(value, f"metric {metric.name} attribute {key}") + assert not leaks, "forbidden values leaked into telemetry:\n" + "\n".join( + sorted(leaks) + ) + + +def assert_package_never_imports_sdk(*module_names): + """ + Import the named modules in a fresh interpreter and assert none of them + imported `opentelemetry.sdk`. + + Run the test that calls this early in your suite: on macOS with CPython + 3.13, starting a subprocess from a process with many threads can crash. + """ + imports = "; ".join(f"import {name}" for name in module_names) + code = ( + f"import sys; {imports}; " + "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert result.stdout.strip() == "[]", ( + f"importing {module_names} pulled in the OpenTelemetry SDK: " + f"{result.stdout.strip()}" + ) diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html index 53a824f1..21a2ea8f 100644 --- a/datasette/templates/_permission_ui_styles.html +++ b/datasette/templates/_permission_ui_styles.html @@ -6,8 +6,20 @@ padding: 1.5em; margin-bottom: 2em; } +.permission-form form { + max-width: 60rem; +} +.permission-form-grid { + display: grid; + gap: 1.5rem; + grid-template-columns: repeat(2, minmax(0, 1fr)); +} +.permission-form-result { + margin-top: 1rem; + max-width: 60rem; +} .form-section { - margin-bottom: 1em; + margin-bottom: 1.25em; } .form-section label { display: block; @@ -15,22 +27,51 @@ font-weight: bold; } .form-section input[type="text"], -.form-section select { - width: 100%; - max-width: 500px; - padding: 0.5em; +.form-section input[type="number"], +.form-section select, +.permission-textarea { + background-color: #fff; + border: 1px solid #aaa; + border-radius: 4px; box-sizing: border-box; - border: 1px solid #ccc; - border-radius: 3px; + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08); + color: #222; + font-family: inherit; + font-size: 1rem; + line-height: 1.4; + max-width: none; + width: 100%; +} +.form-section input[type="text"] { + height: 3rem; + padding: 0.6rem 0.75rem; +} +.form-section input[type="number"] { + height: 3rem; + max-width: 7rem; + padding: 0.6rem 0.75rem; +} +.form-section select { + height: 3rem; + padding: 0.6rem 0.75rem; +} +.permission-textarea { + font-family: monospace; + min-height: 12rem; + padding: 0.75rem; + resize: vertical; } .form-section input[type="text"]:focus, -.form-section select:focus { - outline: 2px solid #0066cc; +.form-section input[type="number"]:focus, +.form-section select:focus, +.permission-textarea:focus { border-color: #0066cc; + box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18); + outline: none; } .form-section small { display: block; - margin-top: 0.3em; + margin-top: 0.45em; color: #666; } .form-actions { @@ -142,4 +183,9 @@ text-align: center; color: #666; } +@media only screen and (max-width: 576px) { + .permission-form-grid { + grid-template-columns: minmax(0, 1fr); + } +} diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html index d7203c1e..8e0f486e 100644 --- a/datasette/templates/_permissions_debug_tabs.html +++ b/datasette/templates/_permissions_debug_tabs.html @@ -44,10 +44,10 @@ diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..fda4032c 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -3,29 +3,11 @@ {% block title %}Debug allow rules{% endblock %} {% block extra_head %} +{% include "_permission_ui_styles.html" %} {% endblock %} @@ -38,24 +20,28 @@ p.message-warning {

      Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

      -
      -
      -

      - -
      -
      -

      - -
      -
      - -
      - +
      +
      +
      +
      + + +
      +
      + + +
      +
      +
      + +
      + -{% if error %}

      {{ error }}

      {% endif %} + {% if error %}

      {{ error }}

      {% endif %} -{% if result == "True" %}

      Result: allow

      {% endif %} + {% if result == "True" %}

      Result: allow

      {% endif %} -{% if result == "False" %}

      Result: deny

      {% endif %} + {% if result == "False" %}

      Result: deny

      {% endif %} +
      {% endblock %} diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 4927cb8d..32686af1 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,7 +3,6 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} - {% endblock %} {% block content %} @@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 18288439..e5aa46f3 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -8,6 +8,7 @@ {% endfor %} + {% for url in extra_js_urls %} diff --git a/datasette/templates/debug_actions.html b/datasette/templates/debug_actions.html index 0ef7b329..c9dccaaa 100644 --- a/datasette/templates/debug_actions.html +++ b/datasette/templates/debug_actions.html @@ -9,7 +9,7 @@ {% include "_permissions_debug_tabs.html" %}

      - This Datasette instance has registered {{ data|length }} action{{ data|length != 1 and "s" or "" }}. + This Datasette instance has registered {{ data.actions|length }} action{{ data.actions|length != 1 and "s" or "" }}. Actions are used by the permission system to control access to different features.

      @@ -26,7 +26,7 @@       - {% for action in data %} + {% for action in data.actions %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 4f8106b8..c73cdfb7 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,7 +3,6 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -49,7 +48,7 @@
      - + Number of results per page (max 200)
      @@ -88,7 +87,7 @@ const hasDebugPermission = {{ 'true' if has_debug_permission else 'false' }}; (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('page'); + const page = params.get('_page'); if (action) { fetchResults(page ? parseInt(page) : 1); } @@ -102,14 +101,14 @@ async function fetchResults(page = 1) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== 'page_size') { + if (value && key !== '_size' && key !== '_page') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('page', page.toString()); - params.append('page_size', pageSize); + params.append('_page', page.toString()); + params.append('_size', pageSize); try { const response = await fetch('{{ urls.path("-/allowed.json") }}?' + params.toString(), { @@ -198,7 +197,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } function displayError(data) { @@ -208,7 +207,7 @@ function displayError(data) { resultsContent.innerHTML = `
      Error: ${escapeHtml(data.error || 'Unknown error')}
      `; - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index 3b229a25..c0081c66 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,9 +1,8 @@ {% extends "base.html" %} -{% block title %}Permission Check{% endblock %} +{% block title %}Explain a permission decision{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} {% block content %} -

      Permission check

      +

      Explain a permission decision

      {% set current_tab = "check" %} {% include "_permissions_debug_tabs.html" %} -

      Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

      - -{% if request.actor %} -

      Current actor: {{ request.actor.get("id", "anonymous") }}

      -{% else %} -

      Current actor: anonymous (not logged in)

      -{% endif %} +

      Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

      -
      +
      - + + + Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. +
      + +
      + - The permission action to check + The operation to evaluate
      -
      - +
      + - For database-level permissions, specify the database name + The database or other parent resource
      -
      - - - For table-level permissions, specify the table name (requires parent) +
      + + + The table, query or other child resource
      - +
      +actionSelect.addEventListener('change', updateResourceFields); +(function initializeFromUrl() { + const params = populateFormFromURL(); + updateResourceFields(); + if (params.get('action')) { + performCheck(); + } +})(); + {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 4410a677..8b0cbbcf 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Debug permissions{% endblock %} +{% block title %}Permission activity{% endblock %} {% block extra_head %} {% include "_permission_ui_styles.html" %} @@ -20,60 +20,45 @@ .check-action, .check-when, .check-result { font-size: 1.3em; } -textarea { - height: 10em; - width: 95%; - box-sizing: border-box; - padding: 0.5em; - border: 2px dotted black; -} -.two-col { - display: inline-block; - width: 48%; -} -.two-col label { - width: 48%; -} -@media only screen and (max-width: 576px) { - .two-col { - width: 100%; - } -} {% endblock %} {% block content %} -

      Permission playground

      +

      Permission activity

      {% set current_tab = "permissions" %} {% include "_permissions_debug_tabs.html" %} -

      This tool lets you simulate an actor and a permission check for that actor.

      +

      Raw simulator

      + +

      This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

      -
      -
      - - +
      +
      +
      + + +
      -
      -
      -
      - - -
      -
      - - -
      -
      - - +
      +
      + + +
      +
      + + +
      +
      + + +
      @@ -125,7 +110,7 @@ debugPost.addEventListener('submit', function(ev) { }); -

      Recent permissions checks

      +

      Recent permission checks

      {% if filter != "all" %}All{% else %}All{% endif %}, diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html index aafa755d..a74c18f7 100644 --- a/datasette/templates/debug_rules.html +++ b/datasette/templates/debug_rules.html @@ -3,7 +3,6 @@ {% block title %}Permission Rules{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -37,7 +36,7 @@

      - + Number of results per page (max 200)
      @@ -75,7 +74,7 @@ const submitBtn = document.getElementById('submit-btn'); (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('page'); + const page = params.get('_page'); if (action) { fetchResults(page ? parseInt(page) : 1); } @@ -89,14 +88,14 @@ async function fetchResults(page = 1) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== 'page_size') { + if (value && key !== '_size' && key !== '_page') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('page', page.toString()); - params.append('page_size', pageSize); + params.append('_page', page.toString()); + params.append('_size', pageSize); try { const response = await fetch('{{ urls.path("-/rules.json") }}?' + params.toString(), { @@ -185,7 +184,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } function displayError(data) { @@ -195,7 +194,7 @@ function displayError(data) { resultsContent.innerHTML = `
      Error: ${escapeHtml(data.error || 'Unknown error')}
      `; - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } diff --git a/datasette/templates/index.html b/datasette/templates/index.html index 03349279..dabf4804 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -26,8 +26,7 @@ {% if database.show_table_row_counts %}{{ "{:,}".format(database.hidden_table_rows_sum) }} rows in {% endif %}{{ database.hidden_tables_count }} hidden table{% if database.hidden_tables_count != 1 %}s{% endif -%} {% endif -%} {% if database.views_count -%} - {% if database.tables_count or database.hidden_tables_count %}, {% endif -%} - {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %} + , {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %} {% endif %}

      {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}

      diff --git a/datasette/templates/table.html b/datasette/templates/table.html index c2131360..3ce88e35 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -47,11 +47,12 @@ {% endif %} {% if count or human_description_en %} -

      - {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows - {% if allow_execute_sql and query.sql %} count all{% endif %} +

      + {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows + + {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} - {% if human_description_en %}{{ human_description_en }}{% endif %} + {% if human_description_en %}{{ human_description_en }}{% endif %}

      {% endif %} diff --git a/datasette/tokens.py b/datasette/tokens.py index 38a55529..79f840d2 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -10,7 +10,7 @@ from __future__ import annotations import dataclasses import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import itsdangerous @@ -18,6 +18,21 @@ if TYPE_CHECKING: from datasette.app import Datasette +class TokenInvalid(Exception): + """ + Raised by a TokenHandler when a token it recognizes is invalid - + for example a bad signature, malformed payload or expired token. + + Datasette responds to this with an HTTP 401 error. Handlers should + return None instead for tokens they do not recognize at all, so that + other registered handlers get a chance to verify them. + """ + + def __init__(self, message="Invalid token"): + self.message = message + super().__init__(message) + + @dataclasses.dataclass class TokenRestrictions: """ @@ -35,24 +50,24 @@ class TokenRestrictions: database: dict[str, list[str]] = dataclasses.field(default_factory=dict) resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict) - def allow_all(self, action: str) -> "TokenRestrictions": + def allow_all(self, action: str) -> TokenRestrictions: """Allow an action across all databases and resources.""" self.all.append(action) return self - def allow_database(self, database: str, action: str) -> "TokenRestrictions": + def allow_database(self, database: str, action: str) -> TokenRestrictions: """Allow an action on a specific database.""" self.database.setdefault(database, []).append(action) return self def allow_resource( self, database: str, resource: str, action: str - ) -> "TokenRestrictions": + ) -> TokenRestrictions: """Allow an action on a specific resource within a database.""" self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self - def abbreviated(self, datasette: "Datasette") -> Optional[dict]: + def abbreviated(self, datasette: Datasette) -> dict | None: """ Return the abbreviated ``_r`` dictionary shape for this set of restrictions, using action abbreviations registered with ``datasette``. @@ -97,19 +112,23 @@ class TokenHandler: async def create_token( self, - datasette: "Datasette", + datasette: Datasette, actor_id: str, *, - expires_after: Optional[int] = None, - restrictions: Optional[TokenRestrictions] = None, + expires_after: int | None = None, + restrictions: TokenRestrictions | None = None, ) -> str: """Create and return a token string for the given actor.""" raise NotImplementedError - async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: + async def verify_token(self, datasette: Datasette, token: str) -> dict | None: """ - Verify a token and return an actor dict, or None if this handler - does not recognize the token. + Verify a token and return an actor dict. + + Return None if this handler does not recognize the token at all, + so other handlers can try it. Raise TokenInvalid if the token is + recognized but invalid (bad signature, malformed, expired) - the + request will fail with a 401 error. """ raise NotImplementedError @@ -123,11 +142,11 @@ class SignedTokenHandler(TokenHandler): async def create_token( self, - datasette: "Datasette", + datasette: Datasette, actor_id: str, *, - expires_after: Optional[int] = None, - restrictions: Optional[TokenRestrictions] = None, + expires_after: int | None = None, + restrictions: TokenRestrictions | None = None, ) -> str: if not datasette.setting("allow_signed_tokens"): raise ValueError( @@ -144,32 +163,35 @@ class SignedTokenHandler(TokenHandler): token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) - async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: + async def verify_token(self, datasette: Datasette, token: str) -> dict | None: prefix = "dstok_" - if not datasette.setting("allow_signed_tokens"): + if not token.startswith(prefix): + # Not one of our tokens - leave it for other handlers return None + if not datasette.setting("allow_signed_tokens"): + raise TokenInvalid( + "Signed tokens are not enabled for this Datasette instance" + ) + max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") - if not token.startswith(prefix): - return None - raw = token[len(prefix) :] try: decoded = datasette.unsign(raw, namespace="token") except itsdangerous.BadSignature: - return None + raise TokenInvalid("Invalid token signature") if "t" not in decoded: - return None + raise TokenInvalid("Invalid token: no timestamp") created = decoded["t"] if not isinstance(created, int): - return None + raise TokenInvalid("Invalid token: invalid timestamp") duration = decoded.get("d") if duration is not None and not isinstance(duration, int): - return None + raise TokenInvalid("Invalid token: invalid duration") if (duration is None and max_signed_tokens_ttl) or ( duration is not None @@ -178,9 +200,8 @@ class SignedTokenHandler(TokenHandler): ): duration = max_signed_tokens_ttl - if duration: - if time.time() - created > duration: - return None + if duration and time.time() - created > duration: + raise TokenInvalid("Token has expired") actor = {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/tracer.py b/datasette/tracer.py index 28f3cc09..1fbda6f9 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,10 +1,11 @@ import asyncio +import json +import time +import traceback from contextlib import contextmanager from contextvars import ContextVar + from markupsafe import escape -import time -import json -import traceback tracers = {} @@ -132,17 +133,17 @@ class AsgiTracer: "num_traces": len(traces), "traces": traces, } - try: - content_type = [ + content_type = next( + ( v.decode("utf8") for k, v in response_headers if k.lower() == b"content-type" - ][0] - except IndexError: - content_type = "" + ), + "", + ) if "text/html" in content_type and b"" in accumulated_body: extra = escape(json.dumps(trace_info, indent=2)) - extra_html = f"
      {extra}
      ".encode("utf8") + extra_html = f"
      {extra}
      ".encode() accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): data = json.loads(accumulated_body.decode("utf8")) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 16b3d42b..f8da20f3 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,6 +1,7 @@ -from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib +from .utils import PrefixedUrlString, path_with_format, tilde_encode + class Urls: def __init__(self, ds): @@ -8,8 +9,7 @@ class Urls: def path(self, path, format=None): if not isinstance(path, PrefixedUrlString): - if path.startswith("/"): - path = path[1:] + path = path.removeprefix("/") path = self.ds.setting("base_url") + path if format is not None: path = path_with_format(path=path, format=format) @@ -56,6 +56,7 @@ class Urls: return PrefixedUrlString(path) def row_blob(self, database, table, row_path, column): - return self.table(database, table) + "/{}.blob?_blob_column={}".format( - row_path, urllib.parse.quote_plus(column) + return ( + self.table(database, table) + + f"/{row_path}.blob?_blob_column={urllib.parse.quote_plus(column)}" ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index b4ede953..669d04c7 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,28 +1,31 @@ import asyncio -from contextlib import contextmanager -import aiofiles -import click -from collections import OrderedDict, namedtuple, Counter +import base64 +import binascii import copy import dataclasses -import base64 import hashlib import inspect import json -import markupsafe -import mergedeep import os import re +import secrets import shlex +import shutil import tempfile -import typing import time import types -import secrets -import shutil -from typing import Iterable, List, Tuple +import typing import urllib +from collections import Counter, OrderedDict, namedtuple +from collections.abc import Iterable +from contextlib import contextmanager + +import aiofiles +import click +import markupsafe +import mergedeep import yaml + from .shutil_backport import copytree from .sqlite import sqlite3, supports_table_xinfo @@ -35,7 +38,7 @@ if typing.TYPE_CHECKING: class PaginatedResources: """Paginated results from allowed_resources query.""" - resources: List["Resource"] + resources: list["Resource"] next: str | None # Keyset token for next page (None if no more results) _datasette: typing.Any = dataclasses.field(default=None, repr=False) _action: str = dataclasses.field(default=None, repr=False) @@ -82,22 +85,132 @@ class PaginatedResources: # From https://www.sqlite.org/lang_keywords.html -reserved_words = set( - ( - "abort action add after all alter analyze and as asc attach autoincrement " - "before begin between by cascade case cast check collate column commit " - "conflict constraint create cross current_date current_time " - "current_timestamp database default deferrable deferred delete desc detach " - "distinct drop each else end escape except exclusive exists explain fail " - "for foreign from full glob group having if ignore immediate in index " - "indexed initially inner insert instead intersect into is isnull join key " - "left like limit match natural no not notnull null of offset on or order " - "outer plan pragma primary query raise recursive references regexp reindex " - "release rename replace restrict right rollback row savepoint select set " - "table temp temporary then to transaction trigger union unique update using " - "vacuum values view virtual when where with without" - ).split() -) +reserved_words = { + "abort", + "action", + "add", + "after", + "all", + "alter", + "analyze", + "and", + "as", + "asc", + "attach", + "autoincrement", + "before", + "begin", + "between", + "by", + "cascade", + "case", + "cast", + "check", + "collate", + "column", + "commit", + "conflict", + "constraint", + "create", + "cross", + "current_date", + "current_time", + "current_timestamp", + "database", + "default", + "deferrable", + "deferred", + "delete", + "desc", + "detach", + "distinct", + "drop", + "each", + "else", + "end", + "escape", + "except", + "exclusive", + "exists", + "explain", + "fail", + "for", + "foreign", + "from", + "full", + "glob", + "group", + "having", + "if", + "ignore", + "immediate", + "in", + "index", + "indexed", + "initially", + "inner", + "insert", + "instead", + "intersect", + "into", + "is", + "isnull", + "join", + "key", + "left", + "like", + "limit", + "match", + "natural", + "no", + "not", + "notnull", + "null", + "of", + "offset", + "on", + "or", + "order", + "outer", + "plan", + "pragma", + "primary", + "query", + "raise", + "recursive", + "references", + "regexp", + "reindex", + "release", + "rename", + "replace", + "restrict", + "right", + "rollback", + "row", + "savepoint", + "select", + "set", + "table", + "temp", + "temporary", + "then", + "to", + "transaction", + "trigger", + "union", + "unique", + "update", + "using", + "vacuum", + "values", + "view", + "virtual", + "when", + "where", + "with", + "without", +} APT_GET_DOCKERFILE_EXTRAS = r""" RUN apt-get update && \ @@ -157,7 +270,7 @@ functions_marked_as_documented = [] def documented(fn=None, *, label=None): def decorate(fn): - fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) + fn._datasette_docs_label = label or f"internals_utils_{fn.__name__}" functions_marked_as_documented.append(fn) return fn @@ -224,24 +337,71 @@ def compound_keys_after_sql(pks, start_index=0): return "({})".format("\n or\n".join(or_clauses)) +@documented class CustomJSONEncoder(json.JSONEncoder): + """ + The CustomJSONEncoder class handles serialization for objects commonly used by Datasette, + including SQLite cursors and binary blobs. Datasette uses it internally to serve .json endpoints, + and plugins that return JSON can use it to match Datasette's own handling. + + Built-in types (text, numbers, lists, etc) are encoded the same as Python's built-in ``json`` module. + + - ``sqlite3.Row`` becomes a tuple + - ``sqlite3.Cursor`` becomes a list + + Binary blobs are encoded as an object, with the actual data base64-encoded, + like so: :: + + { + "$base64": True, + "encoded": ..., + } + + Example: https://latest.datasette.io/fixtures/binary_data.json + """ + def default(self, obj): if isinstance(obj, sqlite3.Row): return tuple(obj) if isinstance(obj, sqlite3.Cursor): return list(obj) if isinstance(obj, bytes): - # Does it encode to utf8? - try: - return obj.decode("utf8") - except UnicodeDecodeError: - return { - "$base64": True, - "encoded": base64.b64encode(obj).decode("latin1"), - } + return { + "$base64": True, + "encoded": base64.b64encode(obj).decode("latin1"), + } return json.JSONEncoder.default(self, obj) +class WriteJsonValueError(ValueError): + pass + + +def decode_write_json_cell(value): + if not isinstance(value, dict): + return value + keys = set(value.keys()) + if keys == {"$raw"}: + return value["$raw"] + if keys == {"$base64", "encoded"} and value.get("$base64") is True: + encoded = value["encoded"] + if not isinstance(encoded, str): + raise WriteJsonValueError("$base64 encoded value must be a string") + try: + return base64.b64decode(encoded, validate=True) + except binascii.Error as ex: + raise WriteJsonValueError("Invalid $base64 encoded value") from ex + return value + + +def decode_write_json_row(row): + return {key: decode_write_json_cell(value) for key, value in row.items()} + + +def decode_write_json_rows(rows): + return [decode_write_json_row(row) for row in rows] + + @contextmanager def sqlite_timelimit(conn, ms): deadline = time.perf_counter() + (ms / 1000) @@ -312,7 +472,7 @@ disallawed_sql_res = [ ( re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"), "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( - ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) + ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) ), ) ] @@ -408,7 +568,7 @@ def escape_css_string(s): def escape_sqlite(s): - if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): + if _boring_keyword_re.fullmatch(s) and (s.lower() not in reserved_words): return s return '"{}"'.format(s.replace('"', '""')) @@ -486,10 +646,7 @@ CMD {cmd}""".format( else "" ), environment_variables="\n".join( - [ - "ENV {} '{}'".format(key, value) - for key, value in environment_variables.items() - ] + [f"ENV {key} '{value}'" for key, value in environment_variables.items()] ), install_from=" ".join(install), files=" ".join(files), @@ -588,11 +745,11 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall() fks = [] for info in infos: if info is not None: - id, seq, table_name, from_, to_, on_update, on_delete, match = info + id, seq, table_name, from_, to_, _on_update, _on_delete, _match = info fks.append( { "column": from_, @@ -663,7 +820,8 @@ def detect_spatialite(conn): def detect_fts(conn, table): """Detect if table has a corresponding FTS virtual table and return it""" - rows = conn.execute(detect_fts_sql(table)).fetchall() + sql, params = detect_fts_sql(table) + rows = conn.execute(sql, params).fetchall() if len(rows) == 0: return None else: @@ -671,18 +829,26 @@ def detect_fts(conn, table): def detect_fts_sql(table): - return r""" - select name from sqlite_master - where rootpage = 0 - and ( - sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%' - or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%' - or ( - tbl_name = "{table}" - and sql like '%VIRTUAL TABLE%USING FTS%' + escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return ( + r""" + select name from sqlite_master + where rootpage = 0 + and ( + sql like :fts_double_quoted escape char(92) + or sql like :fts_bracket_quoted escape char(92) + or ( + tbl_name = :table + and sql like '%VIRTUAL TABLE%USING FTS%' + ) ) - ) - """.format(table=table.replace("'", "''")) + """, + { + "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%', + "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%", + "table": table, + }, + ) def detect_json1(conn=None): @@ -693,7 +859,7 @@ def detect_json1(conn=None): try: conn.execute("SELECT json('{}')") return True - except Exception: + except sqlite3.Error: return False finally: if close_conn: @@ -773,9 +939,7 @@ def is_url(value): if not value.startswith("http://") and not value.startswith("https://"): return False # Any whitespace at all is invalid - if whitespace_re.search(value): - return False - return True + return not whitespace_re.search(value) css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$") @@ -828,7 +992,9 @@ def module_from_path(path, name): mod.__file__ = path with open(path, "r") as file: code = compile(file.read(), path, "exec", dont_inherit=True) - exec(code, mod.__dict__) + # Executing the file is the whole point - this is how --plugins-dir loads + # plugins and how metadata/config .py files are evaluated + exec(code, mod.__dict__) # noqa: S102 return mod @@ -985,9 +1151,7 @@ def escape_fts(query): query += '"' bits = _escape_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join( - '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits - ) + return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) class MultiParams: @@ -999,7 +1163,7 @@ class MultiParams: data[key], (list, tuple) ), "dictionary data should be a dictionary of key => [list]" self._data = data - elif isinstance(data, list) or isinstance(data, tuple): + elif isinstance(data, (list, tuple)): new_data = {} for item in data: assert ( @@ -1089,9 +1253,7 @@ def _gather_arguments(fn, kwargs): for parameter in parameters: if parameter not in kwargs: raise TypeError( - "{} requires parameters {}, missing: {}".format( - fn, tuple(parameters), set(parameters) - set(kwargs.keys()) - ) + f"{fn} requires parameters {tuple(parameters)}, missing: {set(parameters) - set(kwargs.keys())}" ) call_with.append(kwargs[parameter]) return call_with @@ -1160,9 +1322,9 @@ def resolve_env_secrets(config, environ): """Create copy that recursively replaces {"$env": "NAME"} with values from environ""" if isinstance(config, dict): if list(config.keys()) == ["$env"]: - return environ.get(list(config.values())[0]) + return environ.get(next(iter(config.values()))) elif list(config.keys()) == ["$file"]: - with open(list(config.values())[0]) as fp: + with open(next(iter(config.values()))) as fp: return fp.read() else: return { @@ -1240,29 +1402,38 @@ class StartupError(Exception): pass -_single_line_comment_re = re.compile(r"--.*") -_multi_line_comment_re = re.compile(r"/\*.*?\*/", re.DOTALL) -_single_quote_re = re.compile(r"'(?:''|[^'])*'") -_double_quote_re = re.compile(r'"(?:\"\"|[^"])*"') +# Comments and string literals, matched in a single pass so that whichever +# construct starts first "wins" - this ensures a comment marker inside a string +# literal (or a quote inside a comment) does not confuse the parameter scan. +_comments_and_strings_re = re.compile( + r""" + --[^\n]* # single line comment + | /\*.*?(?:\*/|\Z) # multi line comment, possibly to end-of-input + | '(?:''|[^'])*' # single quoted string ('' escapes a quote) + | "(?:""|[^"])*" # double quoted identifier ("" escapes a quote) + | \[(?:[^\]])*\] # square-bracket quoted identifier + | `(?:``|[^`])*` # backtick quoted identifier + """, + re.DOTALL | re.VERBOSE, +) _named_param_re = re.compile(r":(\w+)") @documented -def named_parameters(sql: str) -> List[str]: +def named_parameters(sql: str) -> list[str]: """ Given a SQL statement, return a list of named parameters that are used in the statement e.g. for ``select * from foo where id=:id`` this would return ``["id"]`` """ - sql = _single_line_comment_re.sub("", sql) - sql = _multi_line_comment_re.sub("", sql) - sql = _single_quote_re.sub("", sql) - sql = _double_quote_re.sub("", sql) + # Strip comments and string literals first so that any ":name" sequences + # inside them are not mistaken for named parameters + sql = _comments_and_strings_re.sub("", sql) # Extract parameters from what is left return _named_param_re.findall(sql) -async def derive_named_parameters(db: "Database", sql: str) -> List[str]: +async def derive_named_parameters(db: "Database", sql: str) -> list[str]: """ This undocumented but stable method exists for backwards compatibility with plugins that were using it before it switched to named_parameters() @@ -1270,6 +1441,54 @@ async def derive_named_parameters(db: "Database", sql: str) -> List[str]: return named_parameters(sql) +def parse_size_limit(value, default, maximum, name="_size"): + """ + Parse a page-size parameter using the same semantics as the table + view's ?_size=: blank means default, "max" means maximum, integers + must be 0 or greater and no larger than maximum. Raises ValueError + with a message suitable for a 400 response. + """ + if value in (None, ""): + return default + if value == "max": + return maximum + try: + size = int(value) + if size < 0: + raise ValueError + except ValueError: + raise ValueError(f"{name} must be a positive integer") + if size > maximum: + raise ValueError(f"{name} must be <= {maximum}") + return size + + +UNSTABLE_API_MESSAGE = ( + "This API is not part of Datasette's stable interface and may change at any time" +) + + +def error_body(messages, status): + """ + The canonical JSON error body used by every Datasette JSON error response: + + {"ok": False, "error": "...", "errors": ["...", ...], "status": 400} + + "error" is all of the messages joined with "; ", "errors" is the full + list, "status" matches the HTTP status code. Callers may add extra + context keys to the returned dictionary but must not remove these four. + """ + if isinstance(messages, str): + messages = [messages] + messages = [str(message) for message in messages] + return { + "ok": False, + "error": "; ".join(messages), + "errors": messages, + "status": status, + } + + def add_cors_headers(headers): headers["Access-Control-Allow-Origin"] = "*" headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" @@ -1298,7 +1517,7 @@ class TildeEncoder(dict): elif b == _space: res = "+" else: - res = "~{:02X}".format(b) + res = f"~{b:02X}" self[b] = res return res @@ -1347,7 +1566,13 @@ async def row_sql_params_pks(db, table, pk_values): if use_rowid: select = "rowid, *" pks = ["rowid"] - wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)] + wheres = [] + for i, pk in enumerate(pks): + escaped_pk = escape_sqlite(pk) + # Preserve the historic always-quoted SQL exposed by _extra=query + if escaped_pk == pk: + escaped_pk = f'"{pk}"' + wheres.append(f"{escaped_pk}=:p{i}") sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}" params = {} for i, pk_value in enumerate(pk_values): @@ -1393,7 +1618,7 @@ def _combine(base: dict, update: dict) -> dict: return base -def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: +def pairs_to_nested_config(pairs: list[tuple[str, typing.Any]]) -> dict: """ Parse a list of key-value pairs into a nested dictionary. """ @@ -1408,7 +1633,7 @@ def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm method = getattr(pm.hook, name, None) - assert method is not None, "No hook found for {}".format(name) + assert method is not None, f"No hook found for {name}" async def inner(): html_bits = [] @@ -1432,7 +1657,7 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> tuple[dict, dict]: """ Move 'plugins' and 'allow' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys @@ -1519,7 +1744,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict: return { k: ( redact(v) - if not any(pattern in k for pattern in key_patterns) + if not any(pattern in k.casefold() for pattern in key_patterns) else "***" ) for k, v in data.items() diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index c7137e6b..297f5ae5 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -29,6 +29,15 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks if TYPE_CHECKING: from datasette.app import Datasette + from datasette.permissions import Action + + +def _child_collation(action: "Action") -> str: + """Match resource identity without changing the spelling returned by SQL.""" + resource_class = action.resource_class + if resource_class is not None and resource_class.case_insensitive_child: + return "NOCASE" + return "BINARY" async def build_allowed_resources_sql( @@ -149,6 +158,7 @@ async def _build_single_action_sql( raise ValueError(f"Unknown action: {action}") # Get base resources SQL from the resource class + child_collation = _child_collation(action_obj) base_resources_sql = await action_obj.resource_class.resources_sql( datasette, actor=actor ) @@ -185,7 +195,7 @@ async def _build_single_action_sql( if permission_sql.sql is None: continue rule_sqls.append(f""" - SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( + SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( {permission_sql.sql} ) """.strip()) @@ -252,88 +262,62 @@ async def _build_single_action_sql( ] ) - # Continue with the cascading logic - query_parts.extend( - [ - "child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", + # Continue with the cascading logic. + # Aggregate the RULES by cascade level (small), rather than grouping + # base x rules (which scales with the number of resources). + def _agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,", + " json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons", + f" FROM all_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["child_agg AS ("] + + _agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "parent_agg AS ("] + + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "global_agg AS ("] + + _agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Add anonymous decision logic if needed if include_is_private: - query_parts.extend( - [ - "anon_child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "anon_parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_decisions AS (", - " SELECT", - " b.parent, b.child,", - " CASE", - " WHEN acl.any_deny = 1 THEN 0", - " WHEN acl.any_allow = 1 THEN 1", - " WHEN apl.any_deny = 1 THEN 0", - " WHEN apl.any_allow = 1 THEN 1", - " WHEN agl.any_deny = 1 THEN 0", - " WHEN agl.any_allow = 1 THEN 1", - " ELSE 0", - " END AS anon_is_allowed", - " FROM base b", - " JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))", - " JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))", - " JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))", - "),", + + def _anon_agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow", + f" FROM anon_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["anon_child_agg AS ("] + + _anon_agg( + f"parent, child COLLATE {child_collation} AS child,", + "parent IS NOT NULL AND child IS NOT NULL", + f"parent, child COLLATE {child_collation}", + ) + + ["),", "anon_parent_agg AS ("] + + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "anon_global_agg AS ("] + + _anon_agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Final decisions @@ -342,31 +326,28 @@ async def _build_single_action_sql( "decisions AS (", " SELECT", " b.parent, b.child,", - " -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level", + " -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level", " -- Priority order:", - " -- 1. Child-level deny (most specific, blocks access)", - " -- 2. Child-level allow (most specific, grants access)", - " -- 3. Parent-level deny (intermediate, blocks access)", - " -- 4. Parent-level allow (intermediate, grants access)", - " -- 5. Global-level deny (least specific, blocks access)", - " -- 6. Global-level allow (least specific, grants access)", + " -- 1. Child-level deny 2. Child-level allow", + " -- 3. Parent-level deny 4. Parent-level allow", + " -- 5. Global-level deny 6. Global-level allow", " -- 7. Default deny (no rules match)", " CASE", - " WHEN cl.any_deny = 1 THEN 0", - " WHEN cl.any_allow = 1 THEN 1", - " WHEN pl.any_deny = 1 THEN 0", - " WHEN pl.any_allow = 1 THEN 1", - " WHEN gl.any_deny = 1 THEN 0", - " WHEN gl.any_allow = 1 THEN 1", + " WHEN ca.any_deny = 1 THEN 0", + " WHEN ca.any_allow = 1 THEN 1", + " WHEN pa.any_deny = 1 THEN 0", + " WHEN pa.any_allow = 1 THEN 1", + " WHEN ga.any_deny = 1 THEN 0", + " WHEN ga.any_allow = 1 THEN 1", " ELSE 0", " END AS is_allowed,", " CASE", - " WHEN cl.any_deny = 1 THEN cl.deny_reasons", - " WHEN cl.any_allow = 1 THEN cl.allow_reasons", - " WHEN pl.any_deny = 1 THEN pl.deny_reasons", - " WHEN pl.any_allow = 1 THEN pl.allow_reasons", - " WHEN gl.any_deny = 1 THEN gl.deny_reasons", - " WHEN gl.any_allow = 1 THEN gl.allow_reasons", + " WHEN ca.any_deny = 1 THEN ca.deny_reasons", + " WHEN ca.any_allow = 1 THEN ca.allow_reasons", + " WHEN pa.any_deny = 1 THEN pa.deny_reasons", + " WHEN pa.any_allow = 1 THEN pa.allow_reasons", + " WHEN ga.any_deny = 1 THEN ga.deny_reasons", + " WHEN ga.any_allow = 1 THEN ga.allow_reasons", " ELSE '[]'", " END AS reason", ] @@ -374,21 +355,34 @@ async def _build_single_action_sql( if include_is_private: query_parts.append( - " , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private" + " , CASE WHEN (" + "CASE" + " WHEN aca.any_deny = 1 THEN 0" + " WHEN aca.any_allow = 1 THEN 1" + " WHEN apa.any_deny = 1 THEN 0" + " WHEN apa.any_allow = 1 THEN 1" + " WHEN aga.any_deny = 1 THEN 0" + " WHEN aga.any_allow = 1 THEN 1" + " ELSE 0 END" + ") = 0 THEN 1 ELSE 0 END AS is_private" ) query_parts.extend( [ " FROM base b", - " JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))", - " JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))", - " JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))", + " LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child", + " LEFT JOIN parent_agg pa ON pa.parent = b.parent", + " CROSS JOIN global_agg ga", ] ) if include_is_private: - query_parts.append( - " JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))" + query_parts.extend( + [ + " LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child", + " LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent", + " CROSS JOIN anon_global_agg aga", + ] ) query_parts.append(")") @@ -398,10 +392,31 @@ async def _build_single_action_sql( # Wrap each restriction_sql in a subquery to avoid operator precedence issues # with UNION ALL inside the restriction SQL statements restriction_intersect = "\nINTERSECT\n".join( - f"SELECT * FROM ({sql})" for sql in restriction_sqls + f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})" + for sql in restriction_sqls ) + # Decompose by NULL-pattern so the final filter can use pure-equality + # EXISTS lookups (satisfiable via automatic indexes) instead of a + # correlated OR-scan over the whole list. query_parts.extend( - [",", "restriction_list AS (", f" {restriction_intersect}", ")"] + [ + ",", + "restriction_list AS (", + f" {restriction_intersect}", + "),", + "restriction_exact AS (", + " SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL", + "),", + "restriction_parent_any AS (", + " SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL", + "),", + "restriction_child_any AS (", + " SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL", + "),", + "restriction_all AS (", + " SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1", + ")", + ] ) # Final SELECT @@ -416,10 +431,11 @@ async def _build_single_action_sql( # Add restriction filter if there are restrictions if restriction_sqls: query_parts.append(""" - AND EXISTS ( - SELECT 1 FROM restriction_list r - WHERE (r.parent = decisions.parent OR r.parent IS NULL) - AND (r.child = decisions.child OR r.child IS NULL) + AND ( + EXISTS (SELECT 1 FROM restriction_all) + OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent) + OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child) + OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child) )""") # Add parent filter if specified @@ -475,6 +491,7 @@ async def build_permission_rules_sql( union_parts = [] all_params = {} restriction_sqls = [] + child_collation = _child_collation(action_obj) for permission_sql in permission_sqls: all_params.update(permission_sql.params or {}) @@ -488,7 +505,7 @@ async def build_permission_rules_sql( continue union_parts.append(f""" - SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( + SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( {permission_sql.sql} ) """.strip()) @@ -559,6 +576,7 @@ async def check_permissions_for_actions( verdicts = {} for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)): + child_collation = _child_collation(datasette.actions[action]) prefix = f"a{i}_" rule_parts = [] restriction_parts = [] @@ -584,7 +602,7 @@ async def check_permissions_for_actions( if sql is None: continue rule_parts.append( - f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)" + f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)" ) if not rule_parts: @@ -618,7 +636,8 @@ async def check_permissions_for_actions( if restriction_parts: # Database-level restrictions (parent, NULL) match all children restriction_intersect = "\nINTERSECT\n".join( - f"SELECT * FROM ({sql})" for sql in restriction_parts + f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})" + for sql in restriction_parts ) ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)") verdict_sql = f"""({verdict_sql}) AND EXISTS ( @@ -673,3 +692,240 @@ async def check_permission_for_resource( child=child, ) return results[action] + + +async def explain_permission_for_resource( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Explain a permission decision for one action and resource. + + This is intended for Datasette's permission debugging tools. It uses the + same ``permission_resources_sql`` hook results and the same resolution + rules as :func:`check_permissions_for_actions`, but also returns the + matching rules, actor restriction results and ``also_requires`` chain. + + The returned dictionary is part of Datasette's unstable debugging API. + """ + + action_obj = datasette.actions.get(action) + if action_obj is None: + raise ValueError(f"Unknown action: {action}") + + explanation = await _explain_single_action( + datasette=datasette, + actor=actor, + action=action, + parent=parent, + child=child, + ) + + required_actions = [] + if action_obj.also_requires: + required = await explain_permission_for_resource( + datasette=datasette, + actor=actor, + action=action_obj.also_requires, + parent=parent, + child=child, + ) + required_actions.append(required) + + explanation["required_actions"] = required_actions + explanation["allowed"] = bool( + explanation["rule_allowed"] + and explanation["restriction_allowed"] + and all(required["allowed"] for required in required_actions) + ) + explanation["summary"] = _permission_explanation_summary(explanation) + return explanation + + +async def _explain_single_action( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Return matching rules and restrictions for a single action.""" + from datasette.utils.permissions import SKIP_PERMISSION_CHECKS + + permission_sqls = await gather_permission_sql_from_hooks( + datasette=datasette, + actor=actor, + action=action, + ) + + if permission_sqls is SKIP_PERMISSION_CHECKS: + return { + "action": action, + "rule_allowed": True, + "restriction_allowed": True, + "winning_scope": "global", + "matched_rules": [ + { + "scope": "global", + "effect": "allow", + "source": "skip_permission_checks", + "reason": "Permission checks were explicitly skipped", + "decisive": True, + "ignored_because": None, + } + ], + "restrictions": [], + } + + db = datasette.get_internal_database() + matched_rules = [] + restrictions = [] + child_collation = _child_collation(datasette.actions[action]) + + for permission_sql in permission_sqls: + params = dict(permission_sql.params or {}) + parent_param = _unused_parameter_name(params, "_explain_parent") + params[parent_param] = parent + child_param = _unused_parameter_name(params, "_explain_child") + params[child_param] = child + + if permission_sql.sql: + rows = await db.execute( + f""" + SELECT parent, child, allow, reason + FROM ({permission_sql.sql}) AS permission_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child COLLATE {child_collation} = :{child_param}) + """, + params, + ) + for row in rows: + specificity = ( + 2 + if row["child"] is not None + else 1 if row["parent"] is not None else 0 + ) + matched_rules.append( + { + "scope": ("resource", "parent", "global")[2 - specificity], + "effect": "allow" if row["allow"] else "deny", + "source": permission_sql.source, + "reason": row["reason"], + "_specificity": specificity, + } + ) + + if permission_sql.restriction_sql: + restriction_row = ( + await db.execute( + f""" + SELECT EXISTS( + SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child COLLATE {child_collation} = :{child_param}) + ) AS resource_is_in_allowlist + """, + params, + ) + ).first() + restriction_allowed = bool(restriction_row[0]) + restrictions.append( + { + "source": permission_sql.source, + "allowed": restriction_allowed, + "reason": params.get("deny") + or ( + "Resource is included in this restriction allowlist" + if restriction_allowed + else "Resource is not included in this restriction allowlist" + ), + } + ) + + matched_rules.sort( + key=lambda rule: ( + -rule["_specificity"], + 0 if rule["effect"] == "deny" else 1, + rule["source"] or "", + rule["reason"] or "", + ) + ) + + if matched_rules: + winning_specificity = matched_rules[0]["_specificity"] + winning_rules = [ + rule + for rule in matched_rules + if rule["_specificity"] == winning_specificity + ] + rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules) + winning_scope = winning_rules[0]["scope"] + else: + winning_specificity = None + rule_allowed = False + winning_scope = None + + for rule in matched_rules: + specificity = rule.pop("_specificity") + if specificity != winning_specificity: + rule["decisive"] = False + rule["ignored_because"] = "A more specific rule matched" + elif not rule_allowed and rule["effect"] == "allow": + rule["decisive"] = False + rule["ignored_because"] = "A deny rule matched at the same scope" + else: + rule["decisive"] = True + rule["ignored_because"] = None + + return { + "action": action, + "rule_allowed": rule_allowed, + "restriction_allowed": all( + restriction["allowed"] for restriction in restrictions + ), + "winning_scope": winning_scope, + "matched_rules": matched_rules, + "restrictions": restrictions, + } + + +def _unused_parameter_name(params: dict, preferred: str) -> str: + """Return a SQL parameter name that is not already in ``params``.""" + candidate = preferred + suffix = 2 + while candidate in params: + candidate = f"{preferred}_{suffix}" + suffix += 1 + return candidate + + +def _permission_explanation_summary(explanation: dict) -> str: + denied_requirement = next( + ( + required + for required in explanation["required_actions"] + if not required["allowed"] + ), + None, + ) + if denied_requirement: + return ( + f"Denied because {explanation['action']} also requires " + f"{denied_requirement['action']}, which was denied." + ) + if not explanation["matched_rules"]: + return "Denied because no permission rule matched this actor and resource." + if not explanation["rule_allowed"]: + return ( + f"Denied by a {explanation['winning_scope']}-level rule. " + "Deny rules take precedence over allow rules at the same scope." + ) + if not explanation["restriction_allowed"]: + return ( + "Denied because the resource is not included in the actor's restrictions." + ) + return f"Allowed by the matching {explanation['winning_scope']}-level rule." diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index e1631b10..2d4a6cff 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,28 +1,30 @@ +import asyncio import json -from typing import Optional -from datasette.utils import MultiParams, calculate_etag, sha256_file +import re +from http.cookies import Morsel, SimpleCookie +from mimetypes import guess_type +from pathlib import Path +from urllib.parse import parse_qs, parse_qsl, urlunparse + +import aiofiles +import aiofiles.os + +from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file from datasette.utils.multipart import ( - parse_form_data, - MultipartParseError, - FormData, - DEFAULT_MAX_FILE_SIZE, - DEFAULT_MAX_REQUEST_SIZE, - DEFAULT_MAX_FIELDS, - DEFAULT_MAX_FILES, - DEFAULT_MAX_PARTS, DEFAULT_MAX_FIELD_SIZE, + DEFAULT_MAX_FIELDS, + DEFAULT_MAX_FILE_SIZE, + DEFAULT_MAX_FILES, DEFAULT_MAX_MEMORY_FILE_SIZE, DEFAULT_MAX_PART_HEADER_BYTES, DEFAULT_MAX_PART_HEADER_LINES, + DEFAULT_MAX_PARTS, + DEFAULT_MAX_REQUEST_SIZE, DEFAULT_MIN_FREE_DISK_BYTES, + FormData, + MultipartParseError, + parse_form_data, ) -from mimetypes import guess_type -from urllib.parse import parse_qs, urlunparse, parse_qsl -from pathlib import Path -from http.cookies import SimpleCookie, Morsel -import aiofiles -import aiofiles.os -import re # Workaround for adding samesite support to pre 3.8 python Morsel._reserved["samesite"] = "SameSite" @@ -67,16 +69,41 @@ class BadRequest(Base400): status = 400 +class PayloadTooLarge(Base400): + status = 413 + + SAMESITE_VALUES = ("strict", "lax", "none") +# Bodies read fully into memory (post_body/post_vars/json) are capped at this +# size unless the max_post_body_bytes setting says otherwise. Kept deliberately +# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk, +# while these bodies are held in RAM and json.loads() can multiply their +# footprint several times over. +DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB + + +class _RequestHeaders(dict): + """Incoming headers with lowercase keys and case-insensitive lookups.""" + + def __getitem__(self, key): + return super().__getitem__(key.lower()) + + def get(self, key, default=None): + return super().get(key.lower(), default) + + def __contains__(self, key): + return super().__contains__(key.lower()) + class Request: - def __init__(self, scope, receive): + def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES): self.scope = scope self.receive = receive + self.max_post_body_bytes = max_post_body_bytes def __repr__(self): - return ''.format(self.method, self.url) + return f'' @property def method(self): @@ -98,10 +125,10 @@ class Request: @property def headers(self): - return { - k.decode("latin-1").lower(): v.decode("latin-1") + return _RequestHeaders( + (k.decode("latin-1").lower(), v.decode("latin-1")) for k, v in self.scope.get("headers") or [] - } + ) @property def host(self): @@ -141,15 +168,43 @@ class Request: def actor(self): return self.scope.get("actor", None) - async def post_body(self): - body = b"" + async def post_body(self, max_bytes=None): + """ + Read the request body fully into memory. + + The body is capped at max_bytes - or self.max_post_body_bytes + (default 2MB, set from the max_post_body_bytes setting for requests + created by Datasette) if max_bytes is not provided. Pass max_bytes=0 + to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded - + oversized bodies are rejected as soon as the limit is passed, without + buffering the rest. + """ + if max_bytes is None: + max_bytes = self.max_post_body_bytes + too_large = PayloadTooLarge( + f"Request body exceeded maximum size of {max_bytes} bytes" + ) + if max_bytes: + # Reject early if the client declares an oversized body + try: + if int(self.headers.get("content-length", "")) > max_bytes: + raise too_large + except ValueError: + # Missing or malformed - the streaming check below still applies + pass + chunks = [] + received = 0 more_body = True while more_body: message = await self.receive() assert message["type"] == "http.request", message - body += message.get("body", b"") + chunk = message.get("body", b"") + received += len(chunk) + if max_bytes and received > max_bytes: + raise too_large + chunks.append(chunk) more_body = message.get("more_body", False) - return body + return b"".join(chunks) async def post_vars(self): body = await self.post_body() @@ -166,7 +221,7 @@ class Request: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: Optional[int] = DEFAULT_MAX_PARTS, + max_parts: int | None = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -259,12 +314,24 @@ class AsgiLifespan: while True: message = await receive() if message["type"] == "lifespan.startup": - for fn in self.on_startup: - await fn() + try: + for fn in self.on_startup: + await fn() + except Exception as e: # noqa: BLE001 + await send( + {"type": "lifespan.startup.failed", "message": str(e)} + ) + return await send({"type": "lifespan.startup.complete"}) elif message["type"] == "lifespan.shutdown": - for fn in self.on_shutdown: - await fn() + try: + for fn in self.on_shutdown: + await fn() + except Exception as e: # noqa: BLE001 + await send( + {"type": "lifespan.shutdown.failed", "message": str(e)} + ) + return await send({"type": "lifespan.shutdown.complete"}) return else: @@ -444,6 +511,8 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): await asgi_send_html(send, "404: File not found", 404) return + # Only the actual static-file handler can bypass dynamic response privacy. + inner_static._datasette_static = True return inner_static @@ -489,9 +558,9 @@ class Response: httponly=False, samesite="lax", ): - assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format( - SAMESITE_VALUES - ) + assert ( + samesite in SAMESITE_VALUES + ), f"samesite should be one of {SAMESITE_VALUES}" cookie = SimpleCookie() cookie[key] = value for prop_name, prop_value in ( @@ -535,6 +604,18 @@ class Response: content_type="application/json; charset=utf-8", ) + @classmethod + def error(cls, messages, status=400, headers=None): + """ + A JSON error response using Datasette's standard error format. + + messages can be a single string or a list of strings. For errors + that should content-negotiate between JSON and HTML, raise + Forbidden, NotFound, BadRequest or DatasetteError instead and let + Datasette's error handling hooks build the response. + """ + return cls.json(error_body(messages, status), status=status, headers=headers) + @classmethod def redirect(cls, path, status=302, headers=None): headers = headers or {} @@ -571,10 +652,23 @@ class AsgiRunOnFirstRequest: self.asgi = asgi self.on_startup = on_startup self._started = False + # Guards against concurrent early requests interleaving with startup: + # without this, several requests could all observe `_started is + # False` and proceed before any of them finish running the hooks. + self._lock = asyncio.Lock() async def __call__(self, scope, receive, send): - if not self._started: - self._started = True - for hook in self.on_startup: - await hook() + # Leave "lifespan" scope events alone - this shim only exists as a + # fallback for hosts that never send them. It wraps AsgiLifespan, so + # if it ran on_startup here too, a startup exception would escape + # before AsgiLifespan's own try/except got a chance to turn it into + # a lifespan.startup.failed message. + if scope["type"] != "lifespan" and not self._started: + async with self._lock: + # Re-check: another request may have finished startup while + # we were waiting for the lock. + if not self._started: + for hook in self.on_startup: + await hook() + self._started = True return await self.asgi(scope, receive, send) diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index c4b64908..0469d7a8 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -13,7 +13,7 @@ Originally shared here: https://www.djangosnippets.org/snippets/1431/ """ -class BaseConverter(object): +class BaseConverter: decimal_digits = "0123456789" def __init__(self, digits): diff --git a/datasette/utils/check_callable.py b/datasette/utils/check_callable.py index a0997d20..e21a769b 100644 --- a/datasette/utils/check_callable.py +++ b/datasette/utils/check_callable.py @@ -1,6 +1,6 @@ import inspect import types -from typing import NamedTuple, Any +from typing import Any, NamedTuple class CallableStatus(NamedTuple): @@ -19,7 +19,7 @@ def check_callable(obj: Any) -> CallableStatus: if isinstance(obj, types.FunctionType): return CallableStatus(True, inspect.iscoroutinefunction(obj)) - if hasattr(obj, "__call__"): + if callable(obj): return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__)) - assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) + assert False, f"obj {obj!r} is somehow callable with no __call__ method" diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index bf172667..0ddeb847 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,9 +1,30 @@ import textwrap -from datasette.utils import table_column_details +from sqlite_utils import Database as SQLiteUtilsDatabase +from sqlite_utils import Migrations -async def init_internal_db(db): - create_tables_sql = textwrap.dedent(""" +from datasette.utils import escape_sqlite, table_column_details + +INTERNAL_DB_SCHEMA_TABLES = { + "catalog_databases", + "catalog_tables", + "catalog_views", + "catalog_columns", + "catalog_indexes", + "catalog_foreign_keys", + "metadata_instance", + "metadata_databases", + "metadata_resources", + "metadata_columns", + "column_types", + "queries", +} + +INTERNAL_DB_SCHEMA_INDEXES = { + "queries_owner_idx", +} + +INTERNAL_DB_SCHEMA_SQL = textwrap.dedent(""" CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, @@ -67,99 +88,101 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name) ); + + CREATE TABLE IF NOT EXISTS metadata_instance ( + key text, + value text, + unique(key) + ); + + CREATE TABLE IF NOT EXISTS metadata_databases ( + database_name text, + key text, + value text, + unique(database_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_resources ( + database_name text, + resource_name text, + key text, + value text, + unique(database_name, resource_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_columns ( + database_name text, + resource_name text, + column_name text, + key text, + value text, + unique(database_name, resource_name, column_name, key) + ); + + CREATE TABLE IF NOT EXISTS column_types ( + database_name TEXT NOT NULL, + resource_name TEXT NOT NULL, + column_name TEXT NOT NULL, + column_type TEXT NOT NULL, + config TEXT, + PRIMARY KEY (database_name, resource_name, column_name) + ); + + CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name) + ); + + CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); """).strip() - await db.execute_write_script(create_tables_sql) - await initialize_metadata_tables(db) -async def initialize_metadata_tables(db): - await db.execute_write_script(textwrap.dedent(""" - CREATE TABLE IF NOT EXISTS metadata_instance ( - key text, - value text, - unique(key) - ); - - CREATE TABLE IF NOT EXISTS metadata_databases ( - database_name text, - key text, - value text, - unique(database_name, key) - ); - - CREATE TABLE IF NOT EXISTS metadata_resources ( - database_name text, - resource_name text, - key text, - value text, - unique(database_name, resource_name, key) - ); - - CREATE TABLE IF NOT EXISTS metadata_columns ( - database_name text, - resource_name text, - column_name text, - key text, - value text, - unique(database_name, resource_name, column_name, key) - ); - - CREATE TABLE IF NOT EXISTS column_types ( - database_name TEXT NOT NULL, - resource_name TEXT NOT NULL, - column_name TEXT NOT NULL, - column_type TEXT NOT NULL, - config TEXT, - PRIMARY KEY (database_name, resource_name, column_name) - ); - - CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) - ); - - CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); - """)) +internal_migrations = Migrations("datasette_internal") -async def populate_schema_tables(internal_db, db): +def _internal_schema_exists(db): + table_names = set(db.table_names()) + if not INTERNAL_DB_SCHEMA_TABLES.issubset(table_names): + return False + index_names = { + row[0] + for row in db.execute("select name from sqlite_master where type = 'index'") + } + return INTERNAL_DB_SCHEMA_INDEXES.issubset(index_names) + + +@internal_migrations(name="0001_initial") +def initial_internal_schema(db): + if _internal_schema_exists(db): + return + db.executescript(INTERNAL_DB_SCHEMA_SQL) + + +async def init_internal_db(db): + def apply_migrations(conn): + internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False)) + + await db.execute_write_fn(apply_migrations, transaction=False) + + +async def populate_schema_tables(internal_db, db, schema_version): database_name = db.name - def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_views WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) - - await internal_db.execute_write_fn(delete_everything) - tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows @@ -184,25 +207,30 @@ async def populate_schema_tables(internal_db, db): columns = table_column_details(conn, table_name) columns_to_insert.extend( { - **{"database_name": database_name, "table_name": table_name}, + "database_name": database_name, + "table_name": table_name, **column._asdict(), } for column in columns ) foreign_keys = conn.execute( - f"PRAGMA foreign_key_list([{table_name}])" + f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" ).fetchall() foreign_keys_to_insert.extend( { - **{"database_name": database_name, "table_name": table_name}, + "database_name": database_name, + "table_name": table_name, **dict(foreign_key), } for foreign_key in foreign_keys ) - indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() + indexes = conn.execute( + f"PRAGMA index_list({escape_sqlite(table_name)})" + ).fetchall() indexes_to_insert.extend( { - **{"database_name": database_name, "table_name": table_name}, + "database_name": database_name, + "table_name": table_name, **dict(index), } for index in indexes @@ -223,47 +251,76 @@ async def populate_schema_tables(internal_db, db): indexes_to_insert, ) = await db.execute_fn(collect_info) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - tables_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_views (database_name, view_name, rootpage, sql) - values (?, ?, ?, ?) - """, - views_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + def replace_catalog(conn): + # Delete child rows before their catalog_tables parents so this also + # works if a prepare_connection plugin enables foreign key enforcement. + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + ): + conn.execute( + f"DELETE FROM {table} WHERE database_name = ?", + [database_name], + ) + conn.execute( + """ + INSERT OR REPLACE INTO catalog_databases ( + database_name, path, is_memory, schema_version + ) VALUES (?, ?, ?, ?) + """, + [ + database_name, + str(db.path) if db.path is not None else None, + db.is_memory, + schema_version, + ], ) - """, - columns_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + conn.executemany( + """ + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, ) - """, - foreign_keys_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial + conn.executemany( + """ + INSERT INTO catalog_views (database_name, view_name, rootpage, sql) + values (?, ?, ?, ?) + """, + views_to_insert, ) - """, - indexes_to_insert, - ) + conn.executemany( + """ + INSERT INTO catalog_columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + columns_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + foreign_keys_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + indexes_to_insert, + ) + + await internal_db.execute_write_fn(replace_catalog) diff --git a/datasette/utils/multipart.py b/datasette/utils/multipart.py index cfa77486..182c7ab1 100644 --- a/datasette/utils/multipart.py +++ b/datasette/utils/multipart.py @@ -11,15 +11,10 @@ Supports: import asyncio import shutil import tempfile +from collections.abc import Callable from dataclasses import dataclass, field from typing import ( Any, - Callable, - Dict, - List, - Optional, - Tuple, - Union, ) from urllib.parse import parse_qsl @@ -29,7 +24,7 @@ DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024 # 100MB DEFAULT_MAX_FIELDS = 1000 DEFAULT_MAX_FILES = 100 # If max_parts is not specified, it defaults to max_fields + max_files -DEFAULT_MAX_PARTS: Optional[int] = None +DEFAULT_MAX_PARTS: int | None = None DEFAULT_MAX_FIELD_SIZE = 100 * 1024 # 100KB DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024 # 1MB DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024 # 16KB @@ -40,8 +35,6 @@ DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024 # 50MB class MultipartParseError(Exception): """Raised when multipart parsing fails.""" - pass - @dataclass class UploadedFile: @@ -57,7 +50,7 @@ class UploadedFile: name: str filename: str - content_type: Optional[str] + content_type: str | None size: int _file: tempfile.SpooledTemporaryFile = field(repr=False) @@ -86,7 +79,8 @@ class UploadedFile: def __del__(self): try: self._file.close() - except Exception: + except Exception: # noqa: BLE001, S110 + # __del__ must never raise pass @@ -98,27 +92,27 @@ class FormData: """ def __init__(self): - self._data: List[Tuple[str, Union[str, UploadedFile]]] = [] + self._data: list[tuple[str, str | UploadedFile]] = [] - def append(self, key: str, value: Union[str, UploadedFile]) -> None: + def append(self, key: str, value: str | UploadedFile) -> None: """Add a key-value pair.""" self._data.append((key, value)) - def __getitem__(self, key: str) -> Union[str, UploadedFile]: + def __getitem__(self, key: str) -> str | UploadedFile: """Get the first value for a key.""" for k, v in self._data: if k == key: return v raise KeyError(key) - def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]: + def get(self, key: str, default: Any = None) -> str | UploadedFile | None: """Get the first value for a key, or default if not found.""" try: return self[key] except KeyError: return default - def getlist(self, key: str) -> List[Union[str, UploadedFile]]: + def getlist(self, key: str) -> list[str | UploadedFile]: """Get all values for a key.""" return [v for k, v in self._data if k == key] @@ -142,15 +136,15 @@ class FormData: """Return unique keys.""" return list(self) - def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]: + def items(self) -> list[tuple[str, str | UploadedFile]]: """Return all key-value pairs.""" return list(self._data) - def values(self) -> List[Union[str, UploadedFile]]: + def values(self) -> list[str | UploadedFile]: """Return all values.""" return [v for _, v in self._data] - def _uploaded_files(self) -> List[UploadedFile]: + def _uploaded_files(self) -> list[UploadedFile]: """Return UploadedFile instances contained in this form.""" return [v for _, v in self._data if isinstance(v, UploadedFile)] @@ -163,7 +157,7 @@ class FormData: for uploaded in self._uploaded_files(): try: uploaded.close_sync() - except Exception: + except Exception: # noqa: BLE001, S110 # Best-effort cleanup; ignore close errors pass @@ -172,7 +166,7 @@ class FormData: for uploaded in self._uploaded_files(): try: await uploaded.close() - except Exception: + except Exception: # noqa: BLE001, S110 # Best-effort cleanup; ignore close errors pass @@ -189,13 +183,13 @@ class FormData: await self.aclose() -def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: +def parse_content_disposition(header: str) -> dict[str, str | None]: """ Parse Content-Disposition header value. Returns dict with 'name', 'filename' keys (filename may be None). """ - result: Dict[str, Optional[str]] = {"name": None, "filename": None} + result: dict[str, str | None] = {"name": None, "filename": None} # Split on semicolons, handling quoted strings parts = [] @@ -238,7 +232,8 @@ def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: from urllib.parse import unquote result["filename"] = unquote(encoded, encoding="utf-8") - except Exception: + except Exception: # noqa: BLE001, S110 + # Malformed RFC 5987 filename* - fall back to the plain filename pass continue @@ -250,20 +245,19 @@ def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: if key == "name": result["name"] = value - elif key == "filename": - # Only set if filename* hasn't already set it - if result["filename"] is None: - # Strip path components (security) - # Handle both Unix and Windows paths - value = value.replace("\\", "/") - if "/" in value: - value = value.rsplit("/", 1)[-1] - result["filename"] = value + # Only set filename if filename* hasn't already set it + elif key == "filename" and result["filename"] is None: + # Strip path components (security) + # Handle both Unix and Windows paths + value = value.replace("\\", "/") + if "/" in value: + value = value.rsplit("/", 1)[-1] + result["filename"] = value return result -def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]: +def parse_content_type(header: str) -> tuple[str, dict[str, str]]: """ Parse Content-Type header value. @@ -307,7 +301,7 @@ class MultipartParser: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: Optional[int] = DEFAULT_MAX_PARTS, + max_parts: int | None = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -348,12 +342,12 @@ class MultipartParser: self._tempdir = tempfile.gettempdir() # Current part state - self.current_headers: Dict[str, str] = {} - self.current_file: Optional[tempfile.SpooledTemporaryFile] = None + self.current_headers: dict[str, str] = {} + self.current_file: tempfile.SpooledTemporaryFile | None = None self.current_body = bytearray() - self.current_name: Optional[str] = None - self.current_filename: Optional[str] = None - self.current_content_type: Optional[str] = None + self.current_name: str | None = None + self.current_filename: str | None = None + self.current_content_type: str | None = None def feed(self, chunk: bytes) -> None: """Feed a chunk of data to the parser.""" @@ -364,6 +358,13 @@ class MultipartParser: self.buffer.extend(chunk) self._process() + def close(self) -> None: + """Discard completed uploads and any file still being received.""" + if self.current_file is not None: + self.current_file.close() + self.current_file = None + self.form_data.close() + def _process(self) -> None: """Process buffered data.""" while True: @@ -454,7 +455,7 @@ class MultipartParser: # Parse header try: line_str = line.decode("utf-8", errors="replace") - except Exception: + except UnicodeDecodeError: line_str = line.decode("latin-1") if ":" in line_str: @@ -481,7 +482,9 @@ class MultipartParser: if self.file_count > self.max_files: raise MultipartParseError("Too many files") if self.handle_files: - self.current_file = tempfile.SpooledTemporaryFile( + # Outlives this method - it is filled in across parser callbacks + # and then handed to the UploadedFile the caller consumes + self.current_file = tempfile.SpooledTemporaryFile( # noqa: SIM115 max_size=self.max_memory_file_size ) else: @@ -581,6 +584,9 @@ class MultipartParser: def _finish_part(self) -> None: """Finalize current part and add to form data.""" if self.current_name is None: + if self.current_file is not None: + self.current_file.close() + self.current_file = None return if self.current_filename is not None: @@ -644,7 +650,7 @@ async def parse_form_data( max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: Optional[int] = DEFAULT_MAX_PARTS, + max_parts: int | None = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -726,29 +732,50 @@ async def parse_form_data( batch_target = 64 * 1024 batch = bytearray() + async def run_parser(fn, *args): + # Cancellation must not close files while a worker is using them. + task = asyncio.create_task(asyncio.to_thread(fn, *args)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError as cancelled: + try: + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + continue + task.result() + finally: + raise cancelled + async def flush_batch() -> None: if batch: data = bytes(batch) batch.clear() - await asyncio.to_thread(parser.feed, data) + await run_parser(parser.feed, data) - while True: - message = await receive() - message_type = message.get("type") - if message_type == "http.disconnect": - raise MultipartParseError("Client disconnected during request body") - if message_type is not None and message_type != "http.request": - continue - chunk = message.get("body", b"") - if chunk: - batch.extend(chunk) - if len(batch) >= batch_target: - await flush_batch() - if not message.get("more_body", False): - break + try: + while True: + message = await receive() + message_type = message.get("type") + if message_type == "http.disconnect": + raise MultipartParseError("Client disconnected during request body") + if message_type is not None and message_type != "http.request": + continue + chunk = message.get("body", b"") + if chunk: + batch.extend(chunk) + if len(batch) >= batch_target: + await flush_batch() + if not message.get("more_body", False): + break - await flush_batch() - return await asyncio.to_thread(parser.finalize) + await flush_batch() + return await run_parser(parser.finalize) + except BaseException: + # No FormData is returned to the caller to take ownership on failure. + await asyncio.to_thread(parser.close) + raise else: raise MultipartParseError( diff --git a/datasette/utils/permissions.py b/datasette/utils/permissions.py index fd1e41a1..5a8ee8e2 100644 --- a/datasette/utils/permissions.py +++ b/datasette/utils/permissions.py @@ -2,8 +2,9 @@ from __future__ import annotations import json -from typing import Any, Dict, Iterable, List, Sequence, Tuple import sqlite3 +from collections.abc import Iterable, Sequence +from typing import Any from datasette.permissions import PermissionSQL from datasette.plugins import pm @@ -15,7 +16,7 @@ SKIP_PERMISSION_CHECKS = object() async def gather_permission_sql_from_hooks( *, datasette, actor: dict | None, action: str -) -> List[PermissionSQL] | object: +) -> list[PermissionSQL] | object: """Collect PermissionSQL objects from the permission_resources_sql hook. Ensures that each returned PermissionSQL has a populated ``source``. @@ -34,7 +35,7 @@ async def gather_permission_sql_from_hooks( hookimpls = hook_caller.get_hookimpls() hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action)) - collected: List[PermissionSQL] = [] + collected: list[PermissionSQL] = [] actor_json = json.dumps(actor) if actor is not None else None actor_id = actor.get("id") if isinstance(actor, dict) else None @@ -71,7 +72,7 @@ def _iter_permission_sql_from_result( if isinstance(result, PermissionSQL): return [result] if isinstance(result, (list, tuple)): - collected: List[PermissionSQL] = [] + collected: list[PermissionSQL] = [] for item in result: collected.extend(_iter_permission_sql_from_result(item, action=action)) return collected @@ -90,7 +91,7 @@ def _iter_permission_sql_from_result( def build_rules_union( actor: dict | None, plugins: Sequence[PermissionSQL] -) -> Tuple[str, Dict[str, Any]]: +) -> tuple[str, dict[str, Any]]: """ Compose plugin SQL into a UNION ALL. @@ -102,10 +103,10 @@ def build_rules_union( The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent Plugin parameters should be prefixed with a unique identifier (e.g., source name). """ - parts: List[str] = [] + parts: list[str] = [] actor_json = json.dumps(actor) if actor else None actor_id = actor.get("id") if actor else None - params: Dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} + params: dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} for p in plugins: # No namespacing - just use plugin params as-is @@ -141,10 +142,10 @@ async def resolve_permissions_from_catalog( plugins: Sequence[Any], action: str, candidate_sql: str, - candidate_params: Dict[str, Any] | None = None, + candidate_params: dict[str, Any] | None = None, *, implicit_deny: bool = True, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ Resolve permissions by embedding the provided *candidate_sql* in a CTE. @@ -168,8 +169,8 @@ async def resolve_permissions_from_catalog( - parent, child, allow, reason, source_plugin, depth - resource (rendered "/parent/child" or "/parent" or "/") """ - resolved_plugins: List[PermissionSQL] = [] - restriction_sqls: List[str] = [] + resolved_plugins: list[PermissionSQL] = [] + restriction_sqls: list[str] = [] for plugin in plugins: if callable(plugin) and not isinstance(plugin, PermissionSQL): @@ -398,11 +399,11 @@ async def resolve_permissions_with_candidates( db, actor: dict | None, plugins: Sequence[Any], - candidates: List[Tuple[str, str | None]], + candidates: list[tuple[str, str | None]], action: str, *, implicit_deny: bool = True, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ Resolve permissions without any external candidate table by embedding the candidates as a UNION of parameterized SELECTs in a CTE. @@ -411,8 +412,8 @@ async def resolve_permissions_with_candidates( actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action """ # Build a small CTE for candidates. - cand_rows_sql: List[str] = [] - cand_params: Dict[str, Any] = {} + cand_rows_sql: list[str] = [] + cand_params: dict[str, Any] = {} for i, (parent, child) in enumerate(candidates): pkey = f"cand_p_{i}" ckey = f"cand_c_{i}" diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index d1fd1bd7..d323f5d6 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -6,7 +6,7 @@ https://github.com/python/cpython/blob/v3.8.3/LICENSE """ import os -from shutil import copy, copy2, copystat, Error +from shutil import Error, copy, copy2, copystat def _copytree( diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 0a3a947c..6325e890 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -1,6 +1,8 @@ +import sys from dataclasses import dataclass from typing import Literal +from datasette.utils import escape_sqlite from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type SQLOperation = Literal[ @@ -150,7 +152,6 @@ _SQLITE_INTERNAL_SCHEMA_FUNCTIONS = { "sqlite_rename_test", "substr", } - _AUTHORIZER_ACTION_NAMES = { getattr(sqlite3, name): name for name in ( @@ -196,6 +197,16 @@ def _allow_authorizer_action(*args): return sqlite3.SQLITE_OK +def _disable_authorizer(conn): + # Python 3.11 added support for unregistering an authorizer using None. + # On Python 3.10, None is installed as the callback instead, and the next + # statement fails with "not authorized" when sqlite3 tries to call it. + if sys.version_info >= (3, 11): + conn.set_authorizer(None) + else: + conn.set_authorizer(_allow_authorizer_action) + + def analyze_sql_tables( conn, sql: str, @@ -209,7 +220,9 @@ def analyze_sql_tables( This function is synchronous and connection-based. It temporarily installs a SQLite authorizer, prepares ``EXPLAIN ``, and returns the operation - callbacks observed while SQLite compiles the statement. + callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is + additionally executed inside a rolled-back savepoint so its source-table reads + can be discovered by analyzing a query against the temporary view. """ operations: dict[OperationKey, set[str]] = {} @@ -391,6 +404,10 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK + if action == sqlite3.SQLITE_RECURSIVE: + # Recursive CTE bookkeeping; table reads are reported separately. + return sqlite3.SQLITE_OK + if action == sqlite3.SQLITE_FUNCTION and arg2 is not None: record( "function", @@ -410,12 +427,12 @@ def analyze_sql_tables( database=None, table=None, sqlite_schema=sqlite_schema, - target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, + target=f"{arg1} {arg2}" if arg2 is not None else arg1, source=source, ) return sqlite3.SQLITE_OK - action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) + action_name = _AUTHORIZER_ACTION_NAMES.get(action, f"SQLITE_{action}") record( "unknown", "unknown", @@ -478,24 +495,24 @@ def analyze_sql_tables( conn, key.table, schema=key.sqlite_schema ) finally: - conn.set_authorizer(None) + _disable_authorizer(conn) has_schema_operation = any( key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} for key in operations ) - dropped_tables = { + dropped_tables_and_views = { (key.database, key.table) for key in operations - if key.operation == "drop" and key.target_type == "table" + if key.operation == "drop" and key.target_type in {"table", "view"} } def key_is_drop_table_delete(key: OperationKey) -> bool: return ( key.operation == "delete" and key.target_type == "table" - and (key.database, key.table) in dropped_tables + and (key.database, key.table) in dropped_tables_and_views ) has_user_table_access_in_schema_operation = any( @@ -518,9 +535,7 @@ def analyze_sql_tables( and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS ): return True - if key_is_drop_table_delete(key): - return True - return False + return bool(key_is_drop_table_delete(key)) def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( @@ -531,7 +546,7 @@ def analyze_sql_tables( return None return table_kind_cache[(key.sqlite_schema, key.table)] - return SQLAnalysis( + analysis = SQLAnalysis( operations=tuple( Operation( operation=key.operation, @@ -548,3 +563,58 @@ def analyze_sql_tables( for key, columns in operations.items() ) ) + + # SQLite does not resolve the SELECT body of a view when preparing CREATE + # VIEW, so its authorizer does not report reads from the view's source + # tables. Temporarily create the view, analyze a query against it (which + # does resolve the body), then roll the schema change back. Database-level + # callers use an isolated writable connection for this analysis. + create_view_operations = tuple( + operation + for operation in analysis.operations + if operation.operation == "create" and operation.target_type == "view" + ) + if not create_view_operations: + return analysis + + savepoint = "datasette_analyze_create_view" + conn.execute(f"SAVEPOINT {savepoint}") + try: + conn.execute(sql, params if params is not None else {}) + dependency_reads = [] + for view_operation in create_view_operations: + if view_operation.sqlite_schema is None or view_operation.table is None: + raise sqlite3.OperationalError( + "Could not determine the created view name" + ) + quoted_schema = escape_sqlite(view_operation.sqlite_schema) + quoted_view = escape_sqlite(view_operation.table) + qualified_view = f"{quoted_schema}.{quoted_view}" + view_analysis = analyze_sql_tables( + conn, + f"SELECT * FROM {qualified_view}", + database_name=database_name, + schema_to_database=schema_to_database, + ) + dependency_reads.extend( + operation + for operation in view_analysis.operations + if operation.operation == "read" + and not ( + operation.sqlite_schema == view_operation.sqlite_schema + and operation.table == view_operation.table + ) + ) + finally: + conn.execute(f"ROLLBACK TO {savepoint}") + conn.execute(f"RELEASE {savepoint}") + + existing_operations = set(analysis.operations) + return SQLAnalysis( + operations=analysis.operations + + tuple( + operation + for operation in dependency_reads + if operation not in existing_operations + ) + ) diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 4743ae4c..2ae1be9b 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -15,8 +15,17 @@ if hasattr(sqlite3, "enable_callback_tracebacks"): _cached_sqlite_version = None _cached_supports_returning = None SQLiteTableType = Literal["table", "view", "virtual", "shadow"] +_SQLITE_IDENTIFIER_RE = ( + r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)""" +) _VIRTUAL_TABLE_MODULE_RE = re.compile( - r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", + r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?" + + _SQLITE_IDENTIFIER_RE + + r"(?:\s*\.\s*" + + _SQLITE_IDENTIFIER_RE + + r")?\s*\bUSING\b\s*(" + + _SQLITE_IDENTIFIER_RE + + r")", re.IGNORECASE | re.DOTALL, ) _VIRTUAL_TABLE_SHADOW_SUFFIXES = { @@ -83,24 +92,58 @@ def sqlite_table_type( ) -> SQLiteTableType | None: if supports_table_list(): try: - query = "select type from pragma_table_list where name = ?" - params: tuple[str, ...] = (table,) + # Use the "PRAGMA table_list" statement form rather than the + # pragma_table_list(...) table-valued function. The + # table-valued function is resolved like an ordinary relation + # name, so an attacker-created table or view literally named + # "pragma_table_list" can shadow it and spoof the reported + # type (e.g. claiming a virtual table is an ordinary table). + # The PRAGMA statement form is a distinct piece of SQL syntax + # that always invokes SQLite's built-in pragma, so it cannot + # be shadowed by a user-created relation. if schema is not None: - query += " and schema = ?" - params = (table, schema) - row = conn.execute(query, params).fetchone() - if row is not None and row[0] in {"table", "view", "virtual", "shadow"}: - return row[0] + query = f"PRAGMA {_quote_identifier(schema)}.table_list" + else: + query = "PRAGMA table_list" + cursor = conn.execute(query) + columns = [description[0] for description in cursor.description] + for row in cursor.fetchall(): + record = dict(zip(columns, row)) + if record.get("name") != table: + continue + if schema is not None and record.get("schema") != schema: + continue + row_type = record.get("type") + if row_type in {"table", "view", "virtual", "shadow"}: + return row_type except sqlite3.DatabaseError: pass return _sqlite_table_type_from_schema(conn, table, schema=schema) +def check_structured_write_table(conn, table: str, *, allow_missing=False): + """Validate a row-write target on the connection that will perform the write.""" + # SQLite resolves identifiers case-insensitively. The create API must not + # treat a differently cased existing name as a missing table. + row = conn.execute( + "select name from main.sqlite_master where name = ? collate nocase " + "and type in ('table', 'view')", + (table,), + ).fetchone() + if row is None and allow_missing: + return + if row is not None and sqlite_table_type(conn, row[0]) == "table": + return + # Virtual table modules can interpret row writes as administrative operations. + # Their shadow tables are internal storage, not independently writable data. + raise ValueError("Structured writes require an ordinary table") + + def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]: schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - "select name, sql from {} where type = 'table'".format(schema_table) + f"select name, sql from {schema_table} where type = 'table'" ).fetchall() except sqlite3.DatabaseError: return [] @@ -118,6 +161,63 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str] return sorted(hidden_tables) + content_fts_tables +def sqlite_derived_table_dependencies( + conn, *, schema: str | None = "main" +) -> dict[str, str]: + """Return implementation table -> logical/content table dependencies. + + ``PRAGMA table_list`` safely identifies virtual and shadow tables, but + does not report which virtual table owns a shadow table or which table is + named by an FTS ``content=`` option. Derive those relationships from + ``sqlite_master`` DDL and the documented shadow-table suffixes. + + Database errors propagate: failed discovery must not be mistaken for an + empty dependency map and cached as permission to skip inheritance. + """ + schema_table = _sqlite_schema_table(schema) + rows = conn.execute( + f"select name, sql from {schema_table} where type = 'table'" + ).fetchall() + + table_names = {row[0] for row in rows} + # SQLite identifiers fold ASCII letters only. + identifier_case = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" + ) + canonical_names = {name.translate(identifier_case): name for name in table_names} + dependencies = {} + for virtual_table, sql in rows: + module = _virtual_table_module(sql) + if module is None: + continue + + # SQLite's documented shadow tables are implementation details of + # their logical virtual table. + for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()): + shadow_table = virtual_table + suffix + if shadow_table in table_names: + dependencies[shadow_table] = virtual_table + + # An external-content FTS table can expose values fetched from its + # content table, so it must also depend on that table's permission. + if module in {"fts3", "fts4", "fts5"}: + content_table = _fts_external_content_table(sql) + if content_table: + dependencies[virtual_table] = content_table + + if module in {"fts5vocab", "fts4aux"}: + source = _fts_vocabulary_source(sql, module, schema or "main") + source = ( + canonical_names.get(source.translate(identifier_case)) + if source + else None + ) + # An unresolved source is itself derived, so the one-hop policy denies it. + dependencies[virtual_table] = source or virtual_table + + return dependencies + + def _sqlite_table_type_from_schema( conn, table: str, @@ -127,7 +227,7 @@ def _sqlite_table_type_from_schema( schema_table = _sqlite_schema_table(schema) try: row = conn.execute( - "select type, sql from {} where name = ?".format(schema_table), + f"select type, sql from {schema_table} where name = ?", (table,), ).fetchone() except sqlite3.DatabaseError: @@ -155,7 +255,7 @@ def _is_known_shadow_table( schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - "select name, sql from {} where type = 'table'".format(schema_table) + f"select name, sql from {schema_table} where type = 'table'" ).fetchall() except sqlite3.DatabaseError: return False @@ -174,7 +274,7 @@ def _sqlite_schema_table(schema: str | None) -> str: return "sqlite_master" if schema == "temp": return "sqlite_temp_master" - return "{}.sqlite_master".format(_quote_identifier(schema)) + return f"{_quote_identifier(schema)}.sqlite_master" def _quote_identifier(value: str) -> str: @@ -184,10 +284,151 @@ def _quote_identifier(value: str) -> str: def _virtual_table_module(sql: str | None) -> str | None: if not sql: return None + match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql)) + if match is None: + return None + return _unquote_sql_value(match.group(1)).lower() + + +def _fts_external_content_table(sql: str | None) -> str | None: + """Extract the external ``content=`` table from an FTS declaration.""" + if not sql: + return None + sql = _strip_sql_comments(sql) match = _VIRTUAL_TABLE_MODULE_RE.search(sql) if match is None: return None - return match.group(1).strip("\"'[]`").lower() + open_paren = sql.find("(", match.end()) + if open_paren == -1: + return None + close_paren = sql.rfind(")") + if close_paren <= open_paren: + return None + + for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]): + key, separator, value = argument.partition("=") + if not separator or key.strip().lower() != "content": + continue + return _unquote_sql_value(value.strip()) + return None + + +def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None: + """Resolve a vocabulary source within the current SQLite schema. + + Cross-schema sources cannot be represented by the dependency map and + are conservatively left unresolved. + """ + sql = _strip_sql_comments(sql) + match = _VIRTUAL_TABLE_MODULE_RE.search(sql) + if match is None: + return None + start = sql.find("(", match.end()) + end = sql.rfind(")") + if start < 0 or end <= start: + return None + arguments = [ + _unquote_sql_value(arg.strip()) + for arg in _split_sql_arguments(sql[start + 1 : end]) + ] + expected = 2 if module == "fts5vocab" else 1 + if len(arguments) == expected: + return arguments[0] + if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower(): + return arguments[1] + return None + + +def _split_sql_arguments(arguments: str) -> list[str]: + """Split comma-separated SQLite arguments without splitting quoted text.""" + parts = [] + start = 0 + quote = None + closing_quote = None + index = 0 + while index < len(arguments): + char = arguments[index] + if quote is None: + if char in {"'", '"', "`", "["}: + quote = char + closing_quote = "]" if char == "[" else char + elif char == ",": + parts.append(arguments[start:index]) + start = index + 1 + elif char == closing_quote: + # Single/double/backtick quoting escapes the delimiter by + # doubling it. Square-bracket identifiers do not. + if ( + quote != "[" + and index + 1 < len(arguments) + and arguments[index + 1] == closing_quote + ): + index += 1 + else: + quote = None + closing_quote = None + index += 1 + parts.append(arguments[start:]) + return parts + + +def _strip_sql_comments(sql: str) -> str: + """Remove SQLite comments while preserving quoted strings/identifiers.""" + output = [] + quote = None + closing_quote = None + index = 0 + while index < len(sql): + char = sql[index] + next_char = sql[index + 1] if index + 1 < len(sql) else "" + if quote is None: + if char in {"'", '"', "`", "["}: + quote = char + closing_quote = "]" if char == "[" else char + output.append(char) + elif char == "-" and next_char == "-": + index += 2 + while index < len(sql) and sql[index] not in "\r\n": + index += 1 + output.append(" ") + continue + elif char == "/" and next_char == "*": + index += 2 + while index + 1 < len(sql) and sql[index : index + 2] != "*/": + index += 1 + index = min(index + 2, len(sql)) + output.append(" ") + continue + else: + output.append(char) + else: + output.append(char) + if char == closing_quote: + if ( + quote != "[" + and index + 1 < len(sql) + and sql[index + 1] == closing_quote + ): + output.append(sql[index + 1]) + index += 1 + else: + quote = None + closing_quote = None + index += 1 + return "".join(output) + + +def _unquote_sql_value(value: str) -> str: + if len(value) < 2: + return value + pairs = {"'": "'", '"': '"', "`": "`", "[": "]"} + closing = pairs.get(value[0]) + if closing is None or value[-1] != closing: + return value + unquoted = value[1:-1] + if value[0] != "[": + unquoted = unquoted.replace(closing * 2, closing) + return unquoted def _is_fts_content_virtual_table(sql: str | None) -> bool: diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index de7e94af..e0cb74a7 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -1,9 +1,10 @@ -from asgiref.sync import async_to_sync -from urllib.parse import urlencode import json +from urllib.parse import urlencode + +from asgiref.sync import async_to_sync # These wrapper classes pre-date the introduction of -# datasette.client and httpx to Datasette. They could +# datasette.client and httpx2 to Datasette. They could # be removed if the Datasette tests are modified to # call datasette.client directly. diff --git a/datasette/version.py b/datasette/version.py index 49d270e4..64f28946 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a35" +__version__ = "1.0a41" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index ed7e175f..bac3b39e 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass import dataclasses import types import typing +from dataclasses import dataclass @dataclass(frozen=True) @@ -74,16 +74,14 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - "{}.{} is declared with from_extra() but there is no " - "registered extra of that name".format(cls.__name__, name) + f"{cls.__name__}.{name} is declared with from_extra() but there is no " + "registered extra of that name" ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - "{}.{} is declared with from_extra() but the {} extra is " - "not available for scope {}".format( - cls.__name__, name, name, cls.extras_scope - ) + f"{cls.__name__}.{name} is declared with from_extra() but the {name} extra is " + f"not available for scope {cls.extras_scope}" ) return extra_class.description or "" diff --git a/datasette/views/base.py b/datasette/views/base.py index 30026f4b..7262edb4 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -2,20 +2,20 @@ import csv import hashlib import sys -from datasette.utils.asgi import Request from datasette.utils import ( - add_cors_headers, EscapeHtmlWriter, InvalidSql, LimitedWriter, + add_cors_headers, path_from_row_pks, path_with_format, sqlite3, ) from datasette.utils.asgi import ( AsgiStream, - Response, BadRequest, + Request, + Response, ) @@ -28,12 +28,15 @@ class DatasetteError(Exception): status=500, template=None, message_is_html=False, + plain_message=None, ): self.message = message self.title = title self.error_dict = error_dict or {} self.status = status self.message_is_html = message_is_html + # Plain text used for JSON and CSV error responses when message is HTML + self.plain_message = plain_message class View: @@ -49,9 +52,7 @@ class View: request.path.endswith(".json") or request.headers.get("content-type") == "application/json" ): - response = Response.json( - {"ok": False, "error": "Method not allowed"}, status=405 - ) + response = Response.error("Method not allowed", 405) else: response = Response.text("Method not allowed", status=405) return response @@ -90,9 +91,7 @@ class BaseView: request.path.endswith(".json") or request.headers.get("content-type") == "application/json" ): - response = Response.json( - {"ok": False, "error": "Method not allowed"}, status=405 - ) + response = Response.error("Method not allowed", 405) else: response = Response.text("Method not allowed", status=405) return response @@ -130,12 +129,10 @@ class BaseView: template = environment.select_template(templates) template_context = { **context, - **{ - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - }, + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], } headers = {} if self.has_json_alternate: @@ -152,9 +149,7 @@ class BaseView: template_context["alternate_url_json"] = alternate_url_json headers.update( { - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' } ) return Response.html( @@ -180,18 +175,12 @@ class BaseView: return view -def _error(messages, status=400): - return Response.json({"ok": False, "errors": messages}, status=status) - - async def stream_csv(datasette, fetch_data, request, database): kwargs = {} stream = request.args.get("_stream") # Do not calculate facets or counts: extra_parameters = [ - "{}=1".format(key) - for key in ("_nofacet", "_nocount") - if not request.args.get(key) + f"{key}=1" for key in ("_nofacet", "_nocount") if not request.args.get(key) ] if extra_parameters: # Replace request object with a new one with modified scope @@ -221,9 +210,6 @@ async def stream_csv(datasette, fetch_data, request, database): except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) - except DatasetteError: raise @@ -330,8 +316,9 @@ async def stream_csv(datasette, fetch_data, request, database): else: new_row.append(cell) await writer.writerow(new_row) - except Exception as ex: - sys.stderr.write("Caught this error: {}\n".format(ex)) + except Exception as ex: # noqa: BLE001 + # Streaming CSV: report the error into the response body and stop + sys.stderr.write(f"Caught this error: {ex}\n") sys.stderr.flush() await r.write(str(ex)) return diff --git a/datasette/views/database.py b/datasette/views/database.py index e02de657..da207601 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,48 +1,56 @@ -from dataclasses import asdict, dataclass, field -from urllib.parse import parse_qsl, urlencode import asyncio import hashlib import itertools import json -import markupsafe import os import textwrap +from dataclasses import asdict, dataclass, field +from urllib.parse import parse_qsl, urlencode + +import markupsafe -from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted +from datasette.extras import ExtraScope, extra_names_from_request +from datasette.plugins import pm from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, stored_query_to_dict -from datasette.write_sql import QueryWriteRejected from datasette.utils import ( + InvalidSql, add_cors_headers, await_me_maybe, call_with_supported_arguments, - named_parameters as derive_named_parameters, + error_body, format_bytes, - make_slot_function, - tilde_decode, - to_css_class, - validate_sql_select, is_url, + make_slot_function, path_with_added_args, path_with_format, path_with_removed_args, sqlite3, + tilde_decode, + to_css_class, truncate_url, - InvalidSql, + validate_sql_select, ) -from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden -from datasette.plugins import pm +from datasette.utils import ( + named_parameters as derive_named_parameters, +) +from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response +from datasette.write_sql import QueryWriteRejected +from . import Context from .base import DatasetteError, View, stream_csv -from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .query_helpers import ( + _block_framing, + _ensure_stored_query_execution_permissions, + _table_columns, +) +from .table_create_alter import _create_table_ui_context from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) -from .table_create_alter import _create_table_ui_context -from . import Context @dataclass @@ -99,7 +107,7 @@ class DatabaseView(View): return response if format_ not in ("html", "json"): - raise NotFound("Invalid format: {}".format(format_)) + raise NotFound(f"Invalid format: {format_}") metadata = await datasette.get_database_metadata(database) @@ -163,7 +171,7 @@ class DatabaseView(View): "label": "Create table", "description": "Create a new table in this database.", "attrs": { - "aria-label": "Create table in {}".format(database), + "aria-label": f"Create table in {database}", "data-database-action": "create-table", }, } @@ -270,9 +278,7 @@ class DatabaseView(View): view_name="database", ), headers={ - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' }, ) @@ -325,7 +331,7 @@ class DatabaseContext(Context): database_color: str = field(metadata={"help": "The color assigned to the database"}) database_page_data: dict = field( metadata={ - "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.' + "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.' } ) database_actions: callable = field( @@ -555,7 +561,7 @@ async def database_download(request, datasette): if datasette.cors: add_cors_headers(headers) if db.hash: - etag = '"{}"'.format(db.hash) + etag = f'"{db.hash}"' headers["Etag"] = etag # Has user seen this already? if_none_match = request.headers.get("if-none-match") @@ -607,11 +613,7 @@ class QueryView(View): "_json" ): return Response.json( - { - "ok": False, - "message": ex.message, - "redirect": None, - }, + dict(error_body([ex.message], 403), redirect=None), status=403, ) datasette.add_message(request, ex.message, datasette.ERROR) @@ -646,8 +648,15 @@ class QueryView(View): ok = None redirect_url = None try: + execute_write_kwargs = {"request": request} + if stored_query.is_trusted: + analysis = await db.analyze_sql(stored_query.sql, params_for_query) + if any( + operation.operation == "vacuum" for operation in analysis.operations + ): + execute_write_kwargs["transaction"] = False cursor = await db.execute_write( - stored_query.sql, params_for_query, request=request + stored_query.sql, params_for_query, **execute_write_kwargs ) # success message can come from on_success_message or on_success_message_sql message = None @@ -660,8 +669,9 @@ class QueryView(View): ).first() if message_result: message = message_result[0] - except Exception as ex: - message = "Error running on_success_message_sql: {}".format(ex) + except Exception as ex: # noqa: BLE001 + # Stored-query on_success_message_sql is user-authored + message = f"Error running on_success_message_sql: {ex}" message_type = datasette.ERROR if not message: if stored_query.on_success_message: @@ -675,18 +685,24 @@ class QueryView(View): redirect_url = stored_query.on_success_redirect ok = True - except Exception as ex: + except Exception as ex: # noqa: BLE001 + # Stored-query execution is user-authored SQL message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR redirect_url = stored_query.on_error_redirect ok = False if should_return_json: + if ok: + return Response.json( + { + "ok": True, + "message": message, + "redirect": redirect_url, + } + ) return Response.json( - { - "ok": ok, - "message": message, - "redirect": redirect_url, - } + dict(error_body([message], 400), redirect=redirect_url), + status=400, ) else: datasette.add_message(request, message, message_type) @@ -804,19 +820,23 @@ class QueryView(View): rows = results.rows except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(""" + textwrap.dedent(f"""

      SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

      - + - """.format(markupsafe.escape(ex.sql))).strip(), + """).strip(), title="SQL Interrupted", status=400, message_is_html=True, + plain_message=( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ), ) except sqlite3.DatabaseError as ex: query_error = str(ex) @@ -825,8 +845,6 @@ class QueryView(View): columns = [] except (sqlite3.OperationalError, InvalidSql) as ex: raise DatasetteError(str(ex), title="Invalid SQL", status=400) - except sqlite3.OperationalError as ex: - raise DatasetteError(str(ex)) except DatasetteError: raise @@ -843,14 +861,18 @@ class QueryView(View): raise DatasetteError("?sql= is required", status=400) async def fetch_data_for_csv(request, _next=None): - results = await db.execute(sql, params, truncate=True) + # Reuse the trusted magic parameter values prepared above. + results = await db.execute(sql, params_for_query, truncate=True) data = {"rows": results.rows, "columns": results.columns} return data, None, None return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers.keys(): + elif format_ in datasette.renderers: + if not sql: + raise DatasetteError("?sql= is required", status=400) data = {"ok": True, "rows": rows, "columns": columns} extras = extra_names_from_request(request) + table_extra_registry.validate_requested(extras, ExtraScope.QUERY) if extras: query_extra_context = QueryExtraContext( datasette=datasette, @@ -937,9 +959,7 @@ class QueryView(View): } headers.update( { - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' } ) metadata = await query_metadata() @@ -1020,9 +1040,7 @@ class QueryView(View): + "?" + urlencode( { - **{ - "sql": sql, - }, + "sql": sql, **named_parameter_values, } ) @@ -1124,9 +1142,11 @@ class QueryView(View): headers=headers, ) else: - assert False, "Invalid format: {}".format(format_) + assert False, f"Invalid format: {format_}" if datasette.cors: add_cors_headers(r.headers) + if stored_query_write and format_ == "html": + _block_framing(r) return r @@ -1225,7 +1245,7 @@ async def display_rows(datasette, database, request, rows, columns): '<Binary: {:,} byte{}>'.format( blob_url, ( - ' title="{}"'.format(formatted) + f' title="{formatted}"' if "bytes" not in formatted else "" ), diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index b7e8288e..c4f5e3fe 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -1,15 +1,16 @@ import re from urllib.parse import urlencode +from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource -from datasette.utils import sqlite3 +from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3 from datasette.utils.asgi import Response -from .base import BaseView, _error +from .base import BaseView from .database import display_rows as display_query_rows from .query_helpers import ( - QueryValidationError, SQL_PARAMETER_FORM_PREFIX, + QueryValidationError, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -31,15 +32,7 @@ WRITE_TEMPLATE_LABELS = { "delete": "Delete rows", } WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) -CREATE_TABLE_TEMPLATE_SQL = "\n".join( - ( - "create table new_table (", - " id integer primary key,", - " name text", - " -- created text default (datetime('now'))", - ")", - ) -) +CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" def _parameter_names(columns): @@ -49,11 +42,11 @@ def _parameter_names(columns): base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) base = base.strip("_") or "value" if base[0].isdigit(): - base = "p_{}".format(base) + base = f"p_{base}" name = base index = 2 while name in seen: - name = "{}_{}".format(base, index) + name = f"{base}_{index}" index += 1 seen.add(name) names[column] = name @@ -65,7 +58,7 @@ def _quote_identifier(identifier): def _preferred_where_column(table, columns): - lower_table_id = "{}_id".format(table.lower()) + lower_table_id = f"{table.lower()}_id" return ( next((column for column in columns if column.lower() == "id"), None) or next( @@ -90,17 +83,15 @@ def _insert_template_sql(table, columns): auto_pk = _auto_incrementing_primary_key(columns) insert_columns = [column for column in column_names if column != auto_pk] if not insert_columns: - return "insert into {}\ndefault values".format(_quote_identifier(table)) + return f"insert into {_quote_identifier(table)}\ndefault values" names = _parameter_names(insert_columns) return "\n".join( ( - "insert into {} (".format(_quote_identifier(table)), - ",\n".join( - " {}".format(_quote_identifier(column)) for column in insert_columns - ), + f"insert into {_quote_identifier(table)} (", + ",\n".join(f" {_quote_identifier(column)}" for column in insert_columns), ")", "values (", - ",\n".join(" :{}".format(names[column]) for column in insert_columns), + ",\n".join(f" :{names[column]}" for column in insert_columns), ")", ) ) @@ -114,18 +105,14 @@ def _update_template_sql(table, columns): if not set_columns: return "\n".join( ( - "update {}".format(_quote_identifier(table)), - "set {} = :new_{}".format( - _quote_identifier(where_column), names[where_column] - ), - "where {} = :{}".format( - _quote_identifier(where_column), names[where_column] - ), + f"update {_quote_identifier(table)}", + f"set {_quote_identifier(where_column)} = :new_{names[where_column]}", + f"where {_quote_identifier(where_column)} = :{names[where_column]}", ) ) return "\n".join( ( - "update {}".format(_quote_identifier(table)), + f"update {_quote_identifier(table)}", "set " + ",\n".join( "{}{} = :{}".format( @@ -135,9 +122,7 @@ def _update_template_sql(table, columns): ) for index, column in enumerate(set_columns) ), - "where {} = :{}".format( - _quote_identifier(where_column), names[where_column] - ), + f"where {_quote_identifier(where_column)} = :{names[where_column]}", ) ) @@ -148,10 +133,8 @@ def _delete_template_sql(table, columns): where_column = _preferred_where_column(table, column_names) return "\n".join( ( - "delete from {}".format(_quote_identifier(table)), - "where {} = :{}".format( - _quote_identifier(where_column), names[where_column] - ), + f"delete from {_quote_identifier(table)}", + f"where {_quote_identifier(where_column)} = :{names[where_column]}", ) ) @@ -348,7 +331,7 @@ class ExecuteWriteView(BaseView): ) if not db.is_mutable: return _block_framing( - _error( + Response.error( ["Cannot execute write SQL because this database is immutable."], 403, ) @@ -367,10 +350,10 @@ class ExecuteWriteView(BaseView): actor=request.actor, ): return _block_framing( - _error(["Permission denied: need execute-write-sql"], 403) + Response.error(["Permission denied: need execute-write-sql"], 403) ) if not db.is_mutable: - return _block_framing(_error(["Database is immutable"], 403)) + return _block_framing(Response.error(["Database is immutable"], 403)) data = {} is_json = request.headers.get("content-type", "").startswith("application/json") @@ -384,7 +367,7 @@ class ExecuteWriteView(BaseView): ) except QueryValidationError as ex: if _wants_json(request, is_json, data): - return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.error([ex.message], ex.status)) if ex.flash: self.ds.add_message(request, ex.message, self.ds.ERROR) return await self._render_form( @@ -402,10 +385,10 @@ class ExecuteWriteView(BaseView): try: execute_write_kwargs = {"request": request} cursor = await db.execute_write(sql, params, **execute_write_kwargs) - except sqlite3.DatabaseError as ex: + except (QueryInterrupted, sqlite3.DatabaseError) as ex: message = str(ex) if wants_json: - return _block_framing(_error([message], 400)) + return _block_framing(Response.error([message], 400)) return await self._render_form( request, db, @@ -488,20 +471,18 @@ class ExecuteWriteAnalyzeView(BaseView): actor=request.actor, ): return _block_framing( - _error(["Permission denied: need execute-write-sql"], 403) + Response.error(["Permission denied: need execute-write-sql"], 403) ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - _error( + Response.error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) ) sql = request.args.get("sql") or "" - return _block_framing( - Response.json( - await _execute_write_analysis_data(self.ds, db, sql, request.actor) - ) - ) + analysis = await _execute_write_analysis_data(self.ds, db, sql, request.actor) + analysis["unstable"] = UNSTABLE_API_MESSAGE + return _block_framing(Response.json(analysis)) diff --git a/datasette/views/index.py b/datasette/views/index.py index 6a9462ac..f73ee38a 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -2,10 +2,11 @@ import json from datasette.plugins import pm from datasette.utils import ( + UNSTABLE_API_MESSAGE, + CustomJSONEncoder, add_cors_headers, await_me_maybe, make_slot_function, - CustomJSONEncoder, ) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -45,15 +46,15 @@ class IndexView(BaseView): databases = [] # Iterate over allowed databases instead of all databases - for name in allowed_db_dict.keys(): + for name, allowed_db in allowed_db_dict.items(): db = self.ds.databases[name] - database_private = allowed_db_dict[name].private + database_private = allowed_db.private # Get allowed tables/views for this database allowed_for_db = tables_by_db.get(name, {}) # Get table names from allowed set instead of db.table_names() - table_names = [child_name for child_name in allowed_for_db.keys()] + table_names = [child_name for child_name in allowed_for_db] hidden_table_names = set(await db.hidden_table_names()) @@ -98,7 +99,7 @@ class IndexView(BaseView): # We will be sorting by number of relationships, so populate that field all_foreign_keys = await db.get_all_foreign_keys() for table, foreign_keys in all_foreign_keys.items(): - if table in tables.keys(): + if table in tables: count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) tables[table]["num_relationships_for_sorting"] = count @@ -120,8 +121,7 @@ class IndexView(BaseView): # Only add views if this is less than TRUNCATE_AT if len(tables_and_views_truncated) < TRUNCATE_AT: num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated) - for view in views[:num_views_to_add]: - tables_and_views_truncated.append(view) + tables_and_views_truncated.extend(views[:num_views_to_add]) databases.append( { @@ -151,7 +151,9 @@ class IndexView(BaseView): return Response( json.dumps( { - "databases": {db["name"]: db for db in databases}, + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "databases": databases, "metadata": await self.ds.get_instance_metadata(), }, cls=CustomJSONEncoder, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 026a999f..725d9cdb 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,6 +5,19 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, ) +from datasette.utils import ( + InvalidSql, + escape_sqlite, + parse_size_limit, + path_from_row_pks, + sqlite3, + validate_sql_select, +) +from datasette.utils import ( + named_parameters as derive_named_parameters, +) +from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis from datasette.write_sql import ( IgnoreWriteSqlOperation, QueryWriteRejected, @@ -12,16 +25,6 @@ from datasette.write_sql import ( decision_for_write_sql_operation, operation_is_write, ) -from datasette.utils import ( - named_parameters as derive_named_parameters, - escape_sqlite, - path_from_row_pks, - sqlite3, - validate_sql_select, - InvalidSql, -) -from datasette.utils.asgi import Forbidden -from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -32,7 +35,6 @@ _query_fields = { "hide_sql", "fragment", "parameters", - "params", "is_private", "on_success_message", "on_success_redirect", @@ -91,16 +93,14 @@ def _as_optional_bool(value, name): return True if lowered in {"0", "false", "f", "no", "off"}: return False - raise QueryValidationError("{} must be 0 or 1".format(name)) + raise QueryValidationError(f"{name} must be 0 or 1") -def _query_list_limit(value, default=50): - if value in (None, ""): - return default +def _query_list_limit(value, default, maximum): try: - return min(max(1, int(value)), 1000) + return parse_size_limit(value, default, maximum) except ValueError as ex: - raise QueryValidationError("_size must be an integer") from ex + raise QueryValidationError(str(ex)) from ex def _derived_query_parameters(sql): @@ -173,7 +173,7 @@ async def _json_or_form_payload(request): try: return json.loads(body or b"{}"), True except json.JSONDecodeError as e: - raise QueryValidationError("Invalid JSON: {}".format(e)) + raise QueryValidationError(f"Invalid JSON: {e}") return await request.post_vars(), False @@ -194,7 +194,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + raise QueryValidationError(f"Could not analyze query: {ex}") from ex is_write = _analysis_is_write(analysis) if is_write: @@ -295,8 +295,7 @@ def _coerce_execute_write_payload(data, is_json): for key, value in data.items(): if key in {"sql", "csrftoken", "_json"}: continue - if key.startswith(SQL_PARAMETER_FORM_PREFIX): - key = key[len(SQL_PARAMETER_FORM_PREFIX) :] + key = key.removeprefix(SQL_PARAMETER_FORM_PREFIX) params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") @@ -316,7 +315,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + raise QueryValidationError(f"Could not analyze query: {ex}") from ex if not _analysis_is_write(analysis): raise QueryValidationError( "Use /-/query for read-only SQL; this endpoint only executes writes" @@ -498,7 +497,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): ) try: result = await db.execute( - "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), + f"select {select} from {escape_sqlite(table)} where rowid = ?", [lastrowid], ) except sqlite3.DatabaseError: @@ -541,7 +540,7 @@ async def _prepare_query_create(datasette, request, db, data): raise QueryValidationError("Writable query fields require writable SQL") parameters = _coerce_query_parameters( - data.get("parameters", data.get("params")), + data.get("parameters"), derived, ) return { @@ -586,9 +585,9 @@ async def _prepare_query_update(datasette, request, db, existing: StoredQuery, u actor=request.actor, ) - if "parameters" in update or "params" in update: + if "parameters" in update: parameters = _coerce_query_parameters( - update.get("parameters", update.get("params")), + update.get("parameters"), derived, ) elif "sql" in update: diff --git a/datasette/views/row.py b/datasette/views/row.py index 129216b9..92e75199 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,32 +8,37 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils -from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted -from datasette.events import UpdateRowEvent, DeleteRowEvent +from datasette.events import DeleteRowEvent, UpdateRowEvent +from datasette.extras import ExtraScope, extra_names_from_request +from datasette.plugins import pm from datasette.resources import TableResource -from .base import BaseView, DatasetteError, _error, stream_csv from datasette.utils import ( + CustomJSONEncoder, + CustomRow, + InvalidSql, + WriteJsonValueError, add_cors_headers, await_me_maybe, call_with_supported_arguments, - CustomRow, - InvalidSql, + decode_write_json_row, + escape_sqlite, make_slot_function, path_from_row_pks, - path_with_added_args, path_with_format, path_with_removed_args, - to_css_class, - escape_sqlite, sqlite3, + tilde_decode, + to_css_class, ) -from datasette.plugins import pm -from datasette.extras import extra_names_from_request, ExtraScope +from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response +from datasette.utils.sqlite import check_structured_write_table + from . import Context, from_extra +from .base import BaseView, DatasetteError, stream_csv from .table import ( - display_columns_and_rows, _table_page_data, + display_columns_and_rows, row_label_from_label_column, ) from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -134,6 +139,12 @@ class RowContext(Context): ) +async def _database_and_table_resource_from_request(datasette, request): + db = await datasette.resolve_database(request) + table = tilde_decode(request.url_vars["table"]) + return db, table, TableResource(database=db.name, table=table) + + class RowView(BaseView): name = "row" @@ -185,43 +196,33 @@ class RowView(BaseView): data, extra_template_data, templates = response_or_template_contexts except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(""" + textwrap.dedent(f"""

      SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

      - + - """.format(markupsafe.escape(ex.sql))).strip(), + """).strip(), title="SQL Interrupted", status=400, message_is_html=True, + plain_message=( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ), ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) except DatasetteError: raise end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - # Special case for .jsono extension - redirect to _shape=objects - if format_ == "jsono": - return self.redirect( - request, - path_with_added_args( - request, - {"_shape": "objects"}, - path=request.path.rsplit(".jsono", 1)[0] + ".json", - ), - forward_querystring=False, - ) - - if format_ in self.ds.renderers.keys(): + if format_ in self.ds.renderers: # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -264,13 +265,13 @@ class RowView(BaseView): if status_code is not None: response.status = status_code else: - raise NotFound("Invalid format: {}".format(format_)) + raise NotFound(f"Invalid format: {format_}") ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): ttl = self.ds.setting("default_cache_ttl") - return self.set_response_headers(response, ttl) + return self.set_response_headers(response, ttl, request) async def html(self, request, data, extra_template_data, templates): extras = {} @@ -379,42 +380,54 @@ class RowView(BaseView): view_name=self.name, ), headers={ - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' }, ) - def set_response_headers(self, response, ttl): + def set_response_headers(self, response, ttl, request=None): + private = getattr(request, "_datasette_private_response", False) # Set far-future cache expiry if self.ds.cache_headers and response.status == 200: - ttl = int(ttl) - if ttl == 0: - ttl_header = "no-cache" + if private: + # This response is only visible to the current actor (denied + # to anonymous requests), so it must never be stored by a + # shared cache/CDN - and ?_ttl= must not override that. + response.headers["Cache-Control"] = "private, no-store" + response.headers["Vary"] = "Cookie" else: - ttl_header = f"max-age={ttl}" - response.headers["Cache-Control"] = ttl_header + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" + else: + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header response.headers["Referrer-Policy"] = "no-referrer" if self.ds.cors: add_cors_headers(response.headers) return response async def data(self, request, default_labels=False): - resolved = await self.ds.resolve_row(request) - db = resolved.db + db, table, resource = await _database_and_table_resource_from_request( + self.ds, request + ) database = db.name - table = resolved.table - pk_values = resolved.pk_values - # Ensure user has permission to view this row + # Check the URL resource before resolving the row, so a denied request + # cannot distinguish an existing primary key from a missing one. visible, private = await self.ds.check_visibility( request.actor, action="view-table", - resource=TableResource(database=database, table=table), + resource=resource, ) if not visible: raise Forbidden("You do not have permission to view this table") + # Record whether this response is private (visible to this actor + # only) so set_response_headers() can set appropriate Cache-Control + # headers, regardless of which output format ends up being rendered. + request._datasette_private_response = private + resolved = await self.ds.resolve_row(request) + pk_values = resolved.pk_values results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) @@ -491,8 +504,8 @@ class RowView(BaseView): for row in display_rows: for cell in row: if cell["column"] in pk_set: - cell["value"] = markupsafe.Markup( - "{}".format(cell["value"]) + cell["value"] = markupsafe.Markup("{}").format( + cell["value"] ) label_column = await db.label_column_for_table(table) if is_table else None @@ -506,7 +519,7 @@ class RowView(BaseView): row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = "{} {}".format(pk_path, row_label) + row_action_label = f"{pk_path} {row_label}" row_action_permissions = {} if is_table and db.is_mutable: @@ -519,7 +532,7 @@ class RowView(BaseView): row_actions = [] if row_action_permissions.get("update-row"): attrs = { - "aria-label": "Edit row {}".format(row_action_label), + "aria-label": f"Edit row {row_action_label}", "data-row": row_path, "data-row-action": "edit", } @@ -535,7 +548,7 @@ class RowView(BaseView): ) if row_action_permissions.get("delete-row"): attrs = { - "aria-label": "Delete row {}".format(row_action_label), + "aria-label": f"Delete row {row_action_label}", "data-row": row_path, "data-row-action": "delete", } @@ -565,7 +578,7 @@ class RowView(BaseView): "private": private, "columns": reordered_columns, "foreign_key_tables": await self.foreign_key_tables( - database, table, pk_values + database, table, pk_values, actor=request.actor ), "database_color": db.color, "display_columns": display_columns, @@ -609,6 +622,9 @@ class RowView(BaseView): } extras = extra_names_from_request(request) + if request.url_vars.get("format"): + # Data formats reject unknown extras; HTML ignores them + table_extra_registry.validate_requested(extras, ExtraScope.ROW) # Process extras row_extra_context = RowExtraContext( @@ -639,12 +655,23 @@ class RowView(BaseView): ), ) - async def foreign_key_tables(self, database, table, pk_values): + async def foreign_key_tables(self, database, table, pk_values, *, actor): if len(pk_values) != 1: return [] db = self.ds.databases[database] all_foreign_keys = await db.get_all_foreign_keys() - foreign_keys = all_foreign_keys[table]["incoming"] + foreign_keys = [] + table_permissions = {} + for fk in all_foreign_keys[table]["incoming"]: + other_table = fk["other_table"] + if other_table not in table_permissions: + table_permissions[other_table] = await self.ds.allowed( + action="view-table", + resource=TableResource(database=database, table=other_table), + actor=actor, + ) + if table_permissions[other_table]: + foreign_keys.append(fk) if len(foreign_keys) == 0: return [] @@ -682,7 +709,7 @@ class RowView(BaseView): key, ",".join(pk_values), ) - foreign_key_tables.append({**fk, **{"count": count, "link": link}}) + foreign_key_tables.append({**fk, "count": count, "link": link}) return foreign_key_tables @@ -701,36 +728,57 @@ def _truncated_row_flash_label(label): return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026" -async def _row_flash_message(db, action, resolved, row=None): +async def _row_flash_message( + datasette, request, action, resolved, row=None, *, refresh_row=False +): pk_label = ", ".join(resolved.pk_values) - label_column = await db.label_column_for_table(resolved.table) + # Mutation permission does not grant access to stored row labels. + if not await datasette.allowed( + action="view-table", + resource=TableResource(database=resolved.db.name, table=resolved.table), + actor=request.actor, + ): + return f"{action} row {pk_label}" + + if refresh_row and row is None: + results = await resolved.db.execute( + resolved.sql, resolved.params, truncate=True + ) + row = results.first() + label_column = await resolved.db.label_column_for_table(resolved.table) label = row_label_from_label_column(row or resolved.row, label_column) if label: label = _truncated_row_flash_label(label) if label and label != pk_label: - return "{} row {} ({})".format(action, pk_label, label) - return "{} row {}".format(action, pk_label) + return f"{action} row {pk_label} ({label})" + return f"{action} row {pk_label}" async def _resolve_row_and_check_permission(datasette, request, permission): - from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound + from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound + + try: + _, _, resource = await _database_and_table_resource_from_request( + datasette, request + ) + except DatabaseNotFound as e: + return False, Response.error([f"Database not found: {e.database_name}"], 404) + + # Check the URL resource before resolving the row, so a denied request + # cannot distinguish an existing primary key from a missing one. + if not await datasette.allowed( + action=permission, + resource=resource, + actor=request.actor, + ): + return False, Response.error(["Permission denied"], 403) try: resolved = await datasette.resolve_row(request) - except DatabaseNotFound as e: - return False, _error(["Database not found: {}".format(e.database_name)], 404) except TableNotFound as e: - return False, _error(["Table not found: {}".format(e.table)], 404) + return False, Response.error([f"Table not found: {e.table}"], 404) except RowNotFound as e: - return False, _error(["Record not found: {}".format(e.pk_values)], 404) - - # Ensure user has permission to delete this row - if not await datasette.allowed( - action=permission, - resource=TableResource(database=resolved.db.name, table=resolved.table), - actor=request.actor, - ): - return False, _error(["Permission denied"], 403) + return False, Response.error([f"Record not found: {e.pk_values}"], 404) return True, resolved @@ -750,12 +798,14 @@ class RowDeleteView(BaseView): # Delete table def delete_row(conn): + check_structured_write_table(conn, resolved.table) sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values) try: await resolved.db.execute_write_fn(delete_row, request=request) - except Exception as e: - return _error([str(e)], 500) + except Exception as e: # noqa: BLE001 + # TODO: narrow to expected write errors so Datasette bugs surface as 500s + return Response.error([str(e)], 400) await self.ds.track_event( DeleteRowEvent( @@ -770,7 +820,7 @@ class RowDeleteView(BaseView): table_url = self.ds.urls.table(resolved.db.name, resolved.table) self.ds.add_message( request, - await _row_flash_message(resolved.db, "Deleted", resolved), + await _row_flash_message(self.ds, request, "Deleted", resolved), self.ds.INFO, ) return Response.json({"ok": True, "redirect": str(table_url)}, status=200) @@ -794,18 +844,24 @@ class RowUpdateView(BaseView): try: data = await request.json() except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)]) + return Response.error([f"Invalid JSON: {e}"]) + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if not isinstance(data, dict): - return _error(["JSON must be a dictionary"]) + return Response.error(["JSON must be a dictionary"]) if "update" not in data or not isinstance(data["update"], dict): - return _error(["JSON must contain an update dictionary"]) + return Response.error(["JSON must contain an update dictionary"]) invalid_keys = set(data.keys()) - {"update", "return", "alter"} if invalid_keys: - return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + return Response.error(["Invalid keys: {}".format(", ".join(invalid_keys))]) update = data["update"] + try: + update = decode_write_json_row(update) + except WriteJsonValueError as e: + return Response.error([str(e)], 400) # Validate column types from datasette.views.table import _validate_column_types @@ -814,7 +870,7 @@ class RowUpdateView(BaseView): self.ds, resolved.db.name, resolved.table, [update] ) if ct_errors: - return _error(ct_errors, 400) + return Response.error(ct_errors, 400) alter = data.get("alter") if alter and not await self.ds.allowed( @@ -822,26 +878,35 @@ class RowUpdateView(BaseView): resource=TableResource(database=resolved.db.name, table=resolved.table), actor=request.actor, ): - return _error(["Permission denied for alter-table"], 403) + return Response.error(["Permission denied for alter-table"], 403) def update_row(conn): + check_structured_write_table(conn, resolved.table) sqlite_utils.Database(conn)[resolved.table].update( resolved.pk_values, update, alter=alter ) try: await resolved.db.execute_write_fn(update_row, request=request) - except Exception as e: - return _error([str(e)], 400) + except Exception as e: # noqa: BLE001 + # TODO: narrow to expected write errors so Datasette bugs surface as 500s + return Response.error([str(e)], 400) result = {"ok": True} returned_row = None - if data.get("return"): + # Only read back and disclose the stored row if the actor is also + # allowed to view this table - update-row alone must not be usable + # to read data the actor cannot otherwise see. + if data.get("return") and await self.ds.allowed( + action="view-table", + resource=TableResource(database=resolved.db.name, table=resolved.table), + actor=request.actor, + ): results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) returned_row = results.dicts()[0] - result["row"] = returned_row + result["rows"] = [returned_row] await self.ds.track_event( UpdateRowEvent( @@ -853,18 +918,17 @@ class RowUpdateView(BaseView): ) if request.args.get("_message"): - message_row = returned_row - if message_row is None: - results = await resolved.db.execute( - resolved.sql, resolved.params, truncate=True - ) - message_row = results.first() self.ds.add_message( request, await _row_flash_message( - resolved.db, "Updated", resolved, row=message_row + self.ds, + request, + "Updated", + resolved, + row=returned_row, + refresh_row=True, ), self.ds.INFO, ) - return Response.json(result, status=200) + return Response.json(result, status=200, default=CustomJSONEncoder().default) diff --git a/datasette/views/special.py b/datasette/views/special.py index 3245bc13..72e80316 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,20 +1,25 @@ import json import logging +import secrets +import urllib + +from datasette.events import CreateTokenEvent, LoginEvent, LogoutEvent from datasette.jump import JumpSQL, namespace_sql_params from datasette.plugins import pm -from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource -from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( + UNSTABLE_API_MESSAGE, actor_matches_allow, add_cors_headers, await_me_maybe, - tilde_encode, + error_body, + parse_size_limit, tilde_decode, + tilde_encode, ) +from datasette.utils.asgi import Forbidden, Response + from .base import BaseView, View -import secrets -import urllib logger = logging.getLogger(__name__) @@ -52,9 +57,9 @@ class JsonDataView(BaseView): if self.permission: await self.ds.ensure_permission(action=self.permission, actor=request.actor) if self.needs_request: - data = self.data_callback(request) + data = await await_me_maybe(self.data_callback(request)) else: - data = self.data_callback() + data = await await_me_maybe(self.data_callback()) # Return JSON or HTML depending on format parameter as_format = request.url_vars.get("format") @@ -62,6 +67,8 @@ class JsonDataView(BaseView): headers = {} if self.ds.cors: add_cors_headers(headers) + if isinstance(data, dict): + data = {"ok": True, **data} return Response.json(data, headers=headers) else: context = { @@ -174,9 +181,7 @@ class AutocompleteDebugView(BaseView): ) context.update( { - "autocomplete_url": "{}/-/autocomplete".format( - self.ds.urls.table(database_name, table_name) - ), + "autocomplete_url": f"{self.ds.urls.table(database_name, table_name)}/-/autocomplete", "label_column": await db.label_column_for_table(table_name), } ) @@ -292,6 +297,12 @@ class PermissionsDebugView(BaseView): response, status = await _check_permission_for_actor( self.ds, permission, parent, child, actor ) + if response.get("ok"): + response = { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + **response, + } return Response.json(response, status=status) @@ -300,6 +311,7 @@ class AllowedResourcesView(BaseView): has_json_alternate = False async def get(self, request): + await self.ds.ensure_permission(action="view-instance", actor=request.actor) await self.ds.refresh_schemas() # Check if user has permissions-debug (to show sensitive fields) @@ -348,29 +360,32 @@ class AllowedResourcesView(BaseView): async def _allowed_payload(self, request, has_debug_permission): action = request.args.get("action") if not action: - return {"error": "action parameter is required"}, 400 + return error_body("action parameter is required", 400), 400 if action not in self.ds.actions: - return {"error": f"Unknown action: {action}"}, 404 + return error_body(f"Unknown action: {action}", 404), 404 actor = request.actor if isinstance(request.actor, dict) else None actor_id = actor.get("id") if actor else None parent_filter = request.args.get("parent") child_filter = request.args.get("child") if child_filter and not parent_filter: - return {"error": "parent must be provided when child is specified"}, 400 + return ( + error_body("parent must be provided when child is specified", 400), + 400, + ) try: - page = int(request.args.get("page", "1")) - page_size = int(request.args.get("page_size", "50")) + page = int(request.args.get("_page", "1")) + if page < 1: + raise ValueError except ValueError: - return {"error": "page and page_size must be integers"}, 400 - if page < 1: - return {"error": "page must be >= 1"}, 400 - if page_size < 1: - return {"error": "page_size must be >= 1"}, 400 - max_page_size = 200 - if page_size > max_page_size: - page_size = max_page_size + return error_body("_page must be a positive integer", 400), 400 + try: + page_size = parse_size_limit( + request.args.get("_size"), default=50, maximum=200 + ) + except ValueError as ex: + return error_body(str(ex), 400), 400 offset = (page - 1) * page_size # Use the simplified allowed_resources method @@ -406,10 +421,14 @@ class AllowedResourcesView(BaseView): row["reason"] = resource.reasons allowed_rows.append(row) - except Exception: - # If catalog tables don't exist yet, return empty results + except Exception: # noqa: BLE001 + # Returns empty results if the catalog tables don't exist yet, but + # also swallows the AttributeError raised for instance-level actions + # such as view-instance, which have no resource_class. + # TODO: handle that case explicitly and narrow this to sqlite3.Error return ( { + "ok": True, "action": action, "actor_id": actor_id, "page": page, @@ -434,16 +453,17 @@ class AllowedResourcesView(BaseView): def build_page_url(page_number): pairs = [] for key in request.args: - if key in {"page", "page_size"}: + if key in {"_page", "_size"}: continue for value in request.args.getlist(key): pairs.append((key, value)) - pairs.append(("page", str(page_number))) - pairs.append(("page_size", str(page_size))) + pairs.append(("_page", str(page_number))) + pairs.append(("_size", str(page_size))) query = urllib.parse.urlencode(pairs) return f"{request.path}?{query}" response = { + "ok": True, "action": action, "actor_id": actor_id, "page": page, @@ -485,31 +505,29 @@ class PermissionRulesView(BaseView): # JSON API - action parameter is required action = request.args.get("action") if not action: - return Response.json({"error": "action parameter is required"}, status=400) + return Response.error("action parameter is required", 400) if action not in self.ds.actions: - return Response.json({"error": f"Unknown action: {action}"}, status=404) + return Response.error(f"Unknown action: {action}", 404) actor = request.actor if isinstance(request.actor, dict) else None try: - page = int(request.args.get("page", "1")) - page_size = int(request.args.get("page_size", "50")) + page = int(request.args.get("_page", "1")) + if page < 1: + raise ValueError except ValueError: - return Response.json( - {"error": "page and page_size must be integers"}, status=400 + return Response.error("_page must be a positive integer", 400) + try: + page_size = parse_size_limit( + request.args.get("_size"), default=50, maximum=200 ) - if page < 1: - return Response.json({"error": "page must be >= 1"}, status=400) - if page_size < 1: - return Response.json({"error": "page_size must be >= 1"}, status=400) - max_page_size = 200 - if page_size > max_page_size: - page_size = max_page_size + except ValueError as ex: + return Response.error(str(ex), 400) offset = (page - 1) * page_size from datasette.utils.actions_sql import build_permission_rules_sql - union_sql, union_params, restriction_sqls = await build_permission_rules_sql( + union_sql, union_params, _restriction_sqls = await build_permission_rules_sql( self.ds, actor, action ) await self.ds.refresh_schemas() @@ -555,16 +573,17 @@ class PermissionRulesView(BaseView): def build_page_url(page_number): pairs = [] for key in request.args: - if key in {"page", "page_size"}: + if key in {"_page", "_size"}: continue for value in request.args.getlist(key): pairs.append((key, value)) - pairs.append(("page", str(page_number))) - pairs.append(("page_size", str(page_size))) + pairs.append(("_page", str(page_number))) + pairs.append(("_size", str(page_size))) query = urllib.parse.urlencode(pairs) return f"{request.path}?{query}" response = { + "ok": True, "action": action, "actor_id": (actor or {}).get("id") if actor else None, "page": page, @@ -585,17 +604,17 @@ class PermissionRulesView(BaseView): async def _check_permission_for_actor(ds, action, parent, child, actor): - """Shared logic for checking permissions. Returns a dict with check results.""" + """Shared logic for checking and explaining a permission decision.""" if action not in ds.actions: - return {"error": f"Unknown action: {action}"}, 404 + return error_body(f"Unknown action: {action}", 404), 404 if child and not parent: - return {"error": "parent is required when child is provided"}, 400 + return error_body("parent is required when child is provided", 400), 400 # Use the action's properties to create the appropriate resource object action_obj = ds.actions.get(action) if not action_obj: - return {"error": f"Unknown action: {action}"}, 400 + return error_body(f"Unknown action: {action}", 400), 400 # Global actions (no resource_class) don't have a resource if action_obj.resource_class is None: @@ -610,18 +629,32 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ - return {"error": f"Invalid action configuration: {action}"}, 500 + return error_body(f"Invalid action configuration: {action}", 500), 500 allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor) + from datasette.utils.actions_sql import explain_permission_for_resource + + explanation = await explain_permission_for_resource( + datasette=ds, + actor=actor, + action=action, + parent=parent, + child=child, + ) + response = { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, "action": action, "allowed": bool(allowed), + "actor": actor, "resource": { "parent": parent, "child": child, "path": _resource_path(parent, child), }, + "explanation": explanation, } if actor and "id" in actor: @@ -639,11 +672,25 @@ class PermissionCheckView(BaseView): as_format = request.url_vars.get("format") if not as_format: + actions = [ + { + "name": action.name, + "description": action.description, + "takes_parent": action.takes_parent, + "takes_child": action.takes_child, + "also_requires": action.also_requires, + } + for action in sorted( + self.ds.actions.values(), key=lambda action: action.name + ) + ] return await self.render( ["debug_check.html"], request, { - "sorted_actions": sorted(self.ds.actions.keys()), + "actions": actions, + "actor_json": request.args.get("actor") + or json.dumps(request.actor, indent=2), "has_debug_permission": True, }, ) @@ -651,13 +698,22 @@ class PermissionCheckView(BaseView): # JSON API - action parameter is required action = request.args.get("action") if not action: - return Response.json({"error": "action parameter is required"}, status=400) + return Response.error("action parameter is required", 400) parent = request.args.get("parent") child = request.args.get("child") + actor = request.actor + actor_json = request.args.get("actor") + if actor_json is not None: + try: + actor = json.loads(actor_json) + except json.JSONDecodeError as ex: + return Response.error(f"Invalid actor JSON: {ex}", 400) + if actor is not None and not isinstance(actor, dict): + return Response.error("actor must be a JSON object or null", 400) response, status = await _check_permission_for_actor( - self.ds, action, parent, child, request.actor + self.ds, action, parent, child, actor ) return Response.json(response, status=status) @@ -741,6 +797,8 @@ class CreateTokenView(BaseView): raise Forbidden( "Token authentication cannot be used to create additional tokens" ) + if "_r" in request.actor: + raise Forbidden("Restricted actors cannot create API tokens") async def shared(self, request): self.check_permission(request) @@ -818,6 +876,11 @@ class CreateTokenView(BaseView): else: errors.append("Invalid expire duration unit") + if errors: + context = await self.shared(request) + context["errors"] = errors + return await self.render(["create_token.html"], request, context) + # Are there any restrictions? from datasette.tokens import TokenRestrictions @@ -884,7 +947,7 @@ class ApiExplorerView(BaseView): tables.append({"name": table, "links": table_links}) table_links.append( { - "label": "Get rows for {}".format(table), + "label": f"Get rows for {table}", "method": "GET", "path": self.ds.urls.table(name, table, format="json"), } @@ -904,7 +967,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/insert", "method": "POST", - "label": "Insert rows into {}".format(table), + "label": f"Insert rows into {table}", "json": { "rows": [ { @@ -918,7 +981,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/upsert", "method": "POST", - "label": "Upsert rows into {}".format(table), + "label": f"Upsert rows into {table}", "json": { "rows": [ { @@ -948,7 +1011,7 @@ class ApiExplorerView(BaseView): table_links.append( { "path": self.ds.urls.table(name, table) + "/-/drop", - "label": "Drop table {}".format(table), + "label": f"Drop table {table}", "json": {"confirm": False}, "method": "POST", } @@ -965,7 +1028,7 @@ class ApiExplorerView(BaseView): database_links.append( { "path": self.ds.urls.database(name) + "/-/create", - "label": "Create table in {}".format(name), + "label": f"Create table in {name}", "json": { "table": "new_table", "columns": [ @@ -1198,7 +1261,7 @@ class JumpView(BaseView): match["display_name"] = row["display_name"] matches.append(match) - return Response.json({"matches": matches, "truncated": truncated}) + return Response.json({"ok": True, "matches": matches, "truncated": truncated}) class SchemaBaseView(BaseView): @@ -1206,21 +1269,28 @@ class SchemaBaseView(BaseView): has_json_alternate = False - async def get_database_schema(self, database_name): + async def get_database_schema(self, database_name, actor): """Get schema SQL for a database.""" db = self.ds.databases[database_name] - result = await db.execute( - "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null" + allowed_tables_page = await self.ds.allowed_resources( + "view-table", actor, parent=database_name + ) + allowed_table_names = { + resource.child async for resource in allowed_tables_page.all() + } + result = await db.execute( + "select tbl_name, sql from sqlite_master where sql is not null" + ) + return ";\n".join( + row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names ) - row = result.first() - return row["schema"] if row and row["schema"] else "" def format_json_response(self, data): """Format data as JSON response with CORS headers if needed.""" headers = {} if self.ds.cors: add_cors_headers(headers) - return Response.json(data, headers=headers) + return Response.json({"ok": True, **data}, headers=headers) def format_error_response(self, error_message, format_, status=404): """Format error response based on requested format.""" @@ -1229,7 +1299,7 @@ class SchemaBaseView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response.json( - {"ok": False, "error": error_message}, status=status, headers=headers + error_body(error_message, status), status=status, headers=headers ) else: return Response.text(error_message, status=status) @@ -1275,7 +1345,7 @@ class InstanceSchemaView(SchemaBaseView): # Get schema for each database schemas = [] for database_name in allowed_databases: - schema = await self.get_database_schema(database_name) + schema = await self.get_database_schema(database_name, request.actor) schemas.append({"database": database_name, "schema": schema}) if format_ == "json": @@ -1305,18 +1375,18 @@ class DatabaseSchemaView(SchemaBaseView): database_name = request.url_vars["database"] format_ = request.url_vars.get("format") or "html" - # Check if database exists - if database_name not in self.ds.databases: - return self.format_error_response("Database not found", format_) - - # Check view-database permission + # Permission check comes first, so actors without view-database + # cannot distinguish existing databases from missing ones await self.ds.ensure_permission( action="view-database", resource=DatabaseResource(database=database_name), actor=request.actor, ) - schema = await self.get_database_schema(database_name) + if database_name not in self.ds.databases: + return self.format_error_response("Database not found", format_) + + schema = await self.get_database_schema(database_name, request.actor) if format_ == "json": return self.format_json_response( @@ -1349,10 +1419,14 @@ class TableSchemaView(SchemaBaseView): actor=request.actor, ) + if database_name not in self.ds.databases: + return self.format_error_response("Database not found", format_) + # Get schema for the table db = self.ds.databases[database_name] result = await db.execute( - "select sql from sqlite_master where name = ? and sql is not null", + "select sql from sqlite_master where name = ? " + "and type in ('table', 'view') and sql is not null", [table_name], ) row = result.first() diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 2753f876..03bd9b29 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -2,10 +2,10 @@ from urllib.parse import parse_qsl, urlencode from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict -from datasette.utils import sqlite3, tilde_decode +from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3, tilde_decode from datasette.utils.asgi import Response -from .base import BaseView, _error +from .base import BaseView from .query_helpers import ( QueryValidationError, _as_bool, @@ -34,12 +34,14 @@ class QueryParametersView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + return _block_framing( + Response.error(["Permission denied: need execute-sql"], 403) + ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - _error( + Response.error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) @@ -47,8 +49,16 @@ class QueryParametersView(BaseView): try: parameters = _derived_query_parameters(request.args.get("sql") or "") except QueryValidationError as ex: - return _block_framing(_error([ex.message], ex.status)) - return _block_framing(Response.json({"ok": True, "parameters": parameters})) + return _block_framing(Response.error([ex.message], ex.status)) + return _block_framing( + Response.json( + { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "parameters": parameters, + } + ) + ) def _query_list_url(path, query_string, *, set_args=None, remove_args=None): @@ -82,11 +92,12 @@ class QueryListView(BaseView): limit = _query_list_limit( request.args.get("_size"), default=20 if format_ == "html" else 50, + maximum=self.ds.max_returned_rows, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") is_private = _as_optional_bool(request.args.get("is_private"), "is_private") except QueryValidationError as ex: - return _error([ex.message], ex.status) + return Response.error([ex.message], ex.status) page = await self.ds.list_queries( database, @@ -111,9 +122,9 @@ class QueryListView(BaseView): if key != "_next" ] pairs.append(("_next", page.next)) - next_url = "{}?{}".format( - query_list_path, - urlencode(pairs), + next_url = self.ds.absolute_url( + request, + f"{request.path}?{urlencode(pairs)}", ) current_filters = { @@ -199,7 +210,6 @@ class QueryListView(BaseView): "queries": page.queries, "next": page.next, "next_url": next_url, - "has_more": page.has_more, "limit": page.limit, "show_private_note": any(query.is_private for query in page.queries), "show_trusted_note": any(query.is_trusted for query in page.queries), @@ -269,7 +279,7 @@ class QueryCreateView(BaseView): ), ) response.status = status - return response + return _block_framing(response) async def get(self, request): db = await self.ds.resolve_database(request) @@ -298,28 +308,30 @@ class QueryCreateAnalyzeView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + return _block_framing( + Response.error(["Permission denied: need execute-sql"], 403) + ) if not await self.ds.allowed( action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need store-query"], 403)) + return _block_framing( + Response.error(["Permission denied: need store-query"], 403) + ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - _error( + Response.error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) ) sql = request.args.get("sql") or "" - return _block_framing( - Response.json( - await _query_create_analysis_data(self.ds, db, sql, request.actor) - ) - ) + analysis = await _query_create_analysis_data(self.ds, db, sql, request.actor) + analysis["unstable"] = UNSTABLE_API_MESSAGE + return _block_framing(Response.json(analysis)) class QueryStoreView(QueryCreateView): @@ -346,13 +358,13 @@ class QueryStoreView(QueryCreateView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need execute-sql"], 403) + return Response.error(["Permission denied: need execute-sql"], 403) if not await self.ds.allowed( action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need store-query"], 403) + return Response.error(["Permission denied: need store-query"], 403) is_json = False query_data = {} @@ -369,7 +381,7 @@ class QueryStoreView(QueryCreateView): return await self._error_response( request, db, query_data, ex.message, ex.status ) - return _error([ex.message], ex.status) + return Response.error([ex.message], ex.status) prepared.pop("analysis") name = prepared.pop("name") @@ -378,13 +390,18 @@ class QueryStoreView(QueryCreateView): except sqlite3.IntegrityError as ex: if not is_json and isinstance(query_data, dict): return await self._error_response(request, db, query_data, str(ex), 400) - return _error([str(ex)], 400) + return Response.error([str(ex)], 400) query = await self.ds.get_query(db.name, name) assert query is not None if is_json: return Response.json( - {"ok": True, "query": stored_query_to_dict(query)}, status=201 + { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "query": stored_query_to_dict(query), + }, + status=201, ) self.ds.add_message(request, "Query saved", self.ds.INFO) return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) @@ -398,14 +415,20 @@ class QueryDefinitionView(BaseView): query_name = tilde_decode(request.url_vars["query"]) query = await self.ds.get_query(db.name, query_name) if query is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="view-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied"], 403) - return Response.json({"ok": True, "query": stored_query_to_dict(query)}) + return Response.error(["Permission denied"], 403) + return Response.json( + { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "query": stored_query_to_dict(query), + } + ) class QueryUpdateView(BaseView): @@ -416,15 +439,17 @@ class QueryUpdateView(BaseView): query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied: need update-query"], 403) + return Response.error(["Permission denied: need update-query"], 403) if existing.is_trusted: - return _error(["Trusted queries cannot be updated using the API"], 403) + return Response.error( + ["Trusted queries cannot be updated using the API"], 403 + ) try: data, _ = await _json_or_form_payload(request) @@ -450,7 +475,7 @@ class QueryUpdateView(BaseView): self.ds, request, db, existing, update ) except QueryValidationError as ex: - return _error([ex.message], ex.status) + return Response.error([ex.message], ex.status) await self.ds.update_query(db.name, query_name, **update_kwargs) if data.get("return"): @@ -502,37 +527,37 @@ class QueryEditView(BaseView): ), ) response.status = status - return response + return _block_framing(response) async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) await self.ds.ensure_permission( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ) if existing.is_trusted: - return _error(["Trusted queries cannot be edited"], 403) + return Response.error(["Trusted queries cannot be edited"], 403) return await self._render_form(request, db, existing) async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied: need update-query"], 403) + return Response.error(["Permission denied: need update-query"], 403) if existing.is_trusted: - return _error(["Trusted queries cannot be edited"], 403) + return Response.error(["Trusted queries cannot be edited"], 403) data, _ = await _json_or_form_payload(request) if not isinstance(data, dict): - return _error(["Invalid form submission"], 400) + return Response.error(["Invalid form submission"], 400) sql = data.get("sql") sql = existing.sql if sql is None else sql.strip() title = data.get("title") or "" @@ -604,41 +629,51 @@ class QueryDeleteView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) await self.ds.ensure_permission( action="delete-query", resource=QueryResource(db.name, query_name), actor=request.actor, ) - return await self.render( - ["query_delete.html"], - request, - { - "database": db.name, - "database_color": db.color, - "query": stored_query_to_dict(existing), - "query_url": self.ds.urls.table(db.name, query_name), - }, + if existing.is_trusted: + return Response.error( + ["Trusted queries cannot be deleted using the API"], 403 + ) + return _block_framing( + await self.render( + ["query_delete.html"], + request, + { + "database": db.name, + "database_color": db.color, + "query": stored_query_to_dict(existing), + "query_url": self.ds.urls.table(db.name, query_name), + }, + ) ) async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="delete-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied: need delete-query"], 403) + return Response.error(["Permission denied: need delete-query"], 403) + if existing.is_trusted: + return Response.error( + ["Trusted queries cannot be deleted using the API"], 403 + ) - data, is_json = await _json_or_form_payload(request) + _data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) if is_json: return Response.json({"ok": True}) self.ds.add_message( request, - "Query “{}” deleted".format(existing.title or query_name), + f"Query “{existing.title or query_name}” deleted", self.ds.INFO, ) return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1fc151e6..0a908c69 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,56 +1,67 @@ import asyncio import itertools import json +import time import urllib import urllib.parse +from dataclasses import dataclass, field import markupsafe +import sqlite_utils +from datasette import tracer from datasette.column_types import SQLiteType -from datasette.extras import extra_names_from_request -from datasette.plugins import pm +from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, ) -from datasette.database import QueryInterrupted -from datasette import tracer +from datasette.extras import ExtraScope, extra_names_from_request +from datasette.filters import Filters +from datasette.plugins import pm from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( + CustomJSONEncoder, + CustomRow, + InvalidSql, + WriteJsonValueError, add_cors_headers, + append_querystring, await_me_maybe, call_with_supported_arguments, - CustomRow, - append_querystring, compound_keys_after_sql, - format_bytes, - make_slot_function, - tilde_encode, + decode_write_json_rows, escape_sqlite, filters_should_redirect, + format_bytes, is_url, + make_slot_function, path_from_row_pks, path_with_added_args, path_with_format, path_with_removed_args, path_with_replaced_args, + sqlite3, + tilde_encode, to_css_class, truncate_url, urlsafe_components, value_as_boolean, - InvalidSql, - sqlite3, ) -from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response -from datasette.filters import Filters -import sqlite_utils -from dataclasses import dataclass, field +from datasette.utils.asgi import ( + BadRequest, + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) +from datasette.utils.sqlite import check_structured_write_table -from datasette.extras import ExtraScope from . import Context, from_extra -from .base import BaseView, DatasetteError, _error, stream_csv +from .base import BaseView, DatasetteError, stream_csv from .database import QueryView from .table_create_alter import ( ALTER_TABLE_COLUMN_TYPES, @@ -62,6 +73,7 @@ from .table_create_alter import ( from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, + count_is_truncated, precompute_database_action_permissions, precompute_table_action_permissions, resolve_table_extras, @@ -96,7 +108,6 @@ class TableContext(Context): human_description_en: str = from_extra() is_view: bool = from_extra() metadata: dict = from_extra() - next_url: str = from_extra() primary_keys: list = from_extra() private: bool = from_extra() query: dict = from_extra() @@ -113,6 +124,11 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + next_url: str = field( + metadata={ + "help": "Full URL for the next page of results, or None if there are no more pages. See :ref:`json_api_pagination`." + } + ) count_truncated: bool = field( metadata={ "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." @@ -205,7 +221,7 @@ class TableContext(Context): ) table_insert_ui: dict = field( metadata={ - "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys." + "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys." } ) table_alter_ui: dict = field( @@ -480,8 +496,15 @@ async def _table_insert_ui( ): return None + can_update = await datasette.allowed( + action="update-row", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + column_types_map = await datasette.get_column_types(database_name, table_name) columns = [] + bulk_columns = [] column_details = await db.table_column_details(table_name) for column in column_details: if column.hidden: @@ -492,32 +515,40 @@ async def _table_insert_ui( and len(pks) == 1 and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER ) + column_type = column_types_map.get(column.name) + column_data = { + "name": column.name, + "sqlite_type": _column_sqlite_type_for_insert_form(column), + "notnull": column.notnull, + "default": column.default_value, + "has_default": column.default_value is not None, + "is_pk": is_pk, + "is_auto_pk": is_auto_pk, + "value_kind": _column_value_kind_for_insert_form(column), + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + bulk_columns.append(column_data) if is_auto_pk: continue - column_type = column_types_map.get(column.name) - columns.append( - { - "name": column.name, - "sqlite_type": _column_sqlite_type_for_insert_form(column), - "notnull": column.notnull, - "default": column.default_value, - "has_default": column.default_value is not None, - "is_pk": is_pk, - "value_kind": _column_value_kind_for_insert_form(column), - "column_type": ( - {"type": column_type.name, "config": column_type.config} - if column_type is not None - else None - ), - } - ) + columns.append(column_data) - return { - "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), + data = { + "path": f"{datasette.urls.table(database_name, table_name)}/-/insert", "tableName": table_name, "columns": columns, + "bulkColumns": bulk_columns, "primaryKeys": pks, + "maxInsertRows": datasette.setting("max_insert_rows"), } + if can_update: + data["upsertPath"] = ( + f"{datasette.urls.table(database_name, table_name)}/-/upsert" + ) + return data async def _table_alter_ui( @@ -574,7 +605,7 @@ async def _table_alter_ui( columns.append(column_data) data = { - "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), + "path": f"{datasette.urls.table(database_name, table_name)}/-/alter", "tableName": table_name, "columns": columns, "primaryKeys": pks, @@ -600,9 +631,7 @@ async def _table_alter_ui( actor=request.actor, ) if can_drop_table: - data["dropPath"] = "{}/-/drop".format( - datasette.urls.table(database_name, table_name) - ) + data["dropPath"] = f"{datasette.urls.table(database_name, table_name)}/-/drop" return data @@ -643,7 +672,7 @@ async def display_columns_and_rows( } pks = await db.primary_keys(table_name) pks_for_display = pks - if not pks_for_display: + if not pks_for_display and not await db.view_exists(table_name): pks_for_display = ["rowid"] label_column = None if link_column: @@ -698,12 +727,10 @@ async def display_columns_and_rows( row_label = row_label_from_label_column(row, label_column) row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = "{} {}".format(pk_path, row_label) + row_action_label = f"{pk_path} {row_label}" table_path = datasette.urls.table(database_name, table_name) - row_link = '{flat_pks}'.format( - table_path=table_path, - flat_pks=str(markupsafe.escape(pk_path)), - flat_pks_quoted=row_path, + row_link = ( + f'{markupsafe.escape(pk_path)!s}' ) edit_icon = ( '
      {{ action.name }} {% if action.abbr %}{{ action.abbr }}{% endif %}
      /-/upsert`` API when the actor has both :ref:`insert-row ` and :ref:`update-row ` permissions. (:pr:`2813`) +- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`) +- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text. (:issue:`2806`, :pr:`2822`) +- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. (:issue:`2806`, :pr:`2822`) +- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. +- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits. (:issue:`2823`) +- Row pages for tables with compound primary keys now return a ``400`` error instead of a ``500`` error when the URL row identifier does not contain the correct number of primary key values. Thanks, `Zain Dana Harper `__. (:issue:`2811`, :pr:`2815`) +- The :ref:`execute-write-sql ` interface now supports ``CREATE VIEW`` and ``DROP VIEW`` statements, gated by the new :ref:`create-view ` and :ref:`drop-view ` permissions. (:issue:`2819`, :pr:`2818`) +- Saved-query SQL analysis now handles recursive CTEs, fixing a bug where storing a valid read-only recursive query could be disabled by SQLite's internal ``SQLITE_RECURSIVE`` authorizer callback. (:issue:`2809`, :pr:`2812`) +- ``named_parameters()`` now correctly ignores SQLite comment markers that appear inside string literals, so query forms no longer drop later ``:named`` parameters from SQL such as ``select '--' || :name``. Thanks, `JSap0914 `__. (:pr:`2783`) +- Datasette's internal database schema is now managed using `sqlite-utils migrations `__, using the new dependency on ``sqlite-utils>=4.0``. (:issue:`2827`) +- ``datasette.utils.CustomJSONEncoder`` is now documented as a public API for plugins that need to serialize Datasette values to JSON. Thanks, `Chris Amico `__. (:issue:`1983`, :pr:`1996`) + +This release also includes the results of a `detailed consistency review `__ of Datasette's JSON API in preparation for the 1.0 stable release. Several of these changes are backwards-incompatible with previous 1.0 alphas. The new :ref:`API stability documentation ` describes exactly which parts of the JSON API are covered by the 1.0 stability promise. + +JSON API: breaking changes +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- JSON error responses now use a single canonical format across every endpoint: ``{"ok": false, "error": "...", "errors": [...], "status": 400}``. The ``error`` key joins all error messages together, ``errors`` is the full list of messages and ``status`` always matches the HTTP status code. The legacy ``title`` key is no longer included in JSON errors (it remains available to the HTML error template), and endpoints that previously returned bare ``{"error": ...}`` objects have been updated. See :ref:`json_api_errors`. +- Every JSON object success response now includes ``"ok": true``, including introspection endpoints such as ``/-/versions`` and ``/-/settings``. +- ``/-/plugins.json``, ``/-/databases.json`` and ``/-/actions.json`` now return objects - ``{"ok": true, "plugins": [...]}`` and equivalents - instead of top-level JSON arrays, so these responses can gain additional keys in the future without a breaking change. The ``datasette plugins`` CLI command still outputs a plain array. +- ``/-/databases`` now only lists databases the current actor is allowed to view. It previously listed every attached database, including their filesystem paths, to any actor with ``view-instance``. +- Requests with an invalid or expired ``Authorization: Bearer`` token now receive a ``401`` status with the standard error body and a ``WWW-Authenticate: Bearer error="invalid_token"`` header, instead of being silently treated as unauthenticated. Bearer tokens that no registered token handler recognizes are still ignored, so authentication plugins with their own token formats keep working. Plugin :ref:`token handlers ` can raise the new ``datasette.TokenInvalid`` exception to trigger the same behavior. +- Permission errors for JSON requests now return the standard JSON error format with a ``403`` status. The default forbidden handling previously rendered an HTML error page even for ``.json`` requests. +- ``POST`` to a write canned query now returns a ``400`` error when the SQL fails to execute, instead of a ``200`` status with ``"ok": false`` in the body. The error response includes the standard error keys plus a ``"redirect"`` key. +- The :ref:`row update API ` with ``"return": true`` now responds with a ``"rows"`` list, matching insert and upsert, instead of a singular ``"row"`` object. +- Row delete write failures - such as a constraint violation raised by a trigger - now return ``400`` instead of ``500``, matching the other write endpoints. +- ``//-/query.json`` with a missing or blank ``?sql=`` parameter now returns a ``400`` error, as the CSV format already did, instead of a ``200`` with empty rows. +- Unknown ``?_extra=`` names now return a ``400`` error for JSON and other data formats, instead of being silently ignored. HTML pages continue to ignore unknown names. +- Table JSON responses now include ``next_url`` alongside ``next`` by default - both are ``null`` on the final page. The now-redundant ``?_extra=next_url`` parameter has been removed. +- The stored query list JSON no longer includes ``has_more`` - ``"next": null`` is the end-of-results signal across the whole API. This change also uncovered and fixed a bug where the query list ``next_url`` pointed at the HTML page and was a relative path; it is now an absolute URL that preserves the requested format. +- Stored query JSON objects no longer duplicate the list of parameter names as both ``params`` and ``parameters`` - only ``parameters`` remains. The query create and update APIs no longer accept ``params`` as an input alias either; ``params`` is still the documented key for :ref:`queries defined in configuration `. +- Page size parameters are now consistent across the API: the stored query lists accept ``?_size=max`` and return a ``400`` error for values over the maximum instead of silently clamping them, and the ``/-/allowed`` and ``/-/rules`` permission debug endpoints renamed their ``page`` and ``page_size`` parameters to ``_page`` and ``_size``, matching the underscore grammar used by every other Datasette system parameter. +- ``/-/threads`` now requires the ``permissions-debug`` permission, since it exposes runtime internals such as file paths. It previously only required ``view-instance``. +- Trusted stored queries - those defined in configuration - can no longer be deleted through the JSON API or web interface, matching the existing restriction on editing them. +- The ``//-/schema`` endpoints now check the ``view-database`` permission before checking whether the database exists, so unauthorized actors can no longer probe for the existence of databases. +- SQL time limit errors in JSON responses are now a plain text message. The error string previously embedded an HTML fragment. +- The undocumented homepage JSON at ``/.json`` now returns ``databases`` as a list of objects rather than an object keyed by database name, matching every other collection in the API. +- The legacy ``.jsono`` format extension, long since superseded by ``?_shape=``, has been removed. + +JSON API: other improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The :ref:`write API ` endpoints now parse the request body as JSON regardless of the ``Content-Type`` header, so ``curl -d`` invocations work without remembering to set it. Invalid JSON is a ``400`` error. Cross-site request forgery remains prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` checks. This also fixes a ``500`` error from the insert API when the ``Content-Type`` header was missing entirely. +- New ``Response.error(messages, status=400)`` helper for plugins that need to return a JSON error in Datasette's standard format. See :ref:`internals_response`. +- New ``count_truncated`` extra for table JSON, included automatically whenever ``count`` is requested. ``true`` means the count reached Datasette's counting limit and the real number of rows may be higher. See :ref:`json_api_extra`. +- JSON endpoints that are not part of the documented stable API now declare themselves with an ``"unstable"`` key in their responses. +- New documentation covering the grammar for :ref:`boolean query string arguments `, the reason :ref:`upsert ` returns ``200`` where insert returns ``201``, and advice for plugin authors on :ref:`naming secret configuration keys ` so that ``/-/config`` redacts them automatically. + .. _v1_0_a35: 1.0a35 (2026-06-23) @@ -1010,7 +1207,7 @@ Features - New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) - Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) - ```` is now displayed as ```` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) -- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image `__ has been upgraded to ``3.10.6-slim-bullseye``. (:issue:`1768`) +- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image `__ has been upgraded to ``3.10.6-slim-bullseye``. (:issue:`1768`) - Canned writable queries against immutable databases now show a warning message. (:issue:`1728`) - ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (:pr:`1717`) - ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`) @@ -2039,7 +2236,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you - Removed obsolete ``?_group_count=col`` feature (:issue:`504`) - Improved user interface and documentation for ``datasette publish cloudrun`` (:issue:`608`) - Tables with indexes now show the ``CREATE INDEX`` statements on the table page (:issue:`618`) -- Current version of `uvicorn `__ is now shown on ``/-/versions`` +- Current version of `uvicorn `__ is now shown on ``/-/versions`` - Python 3.8 is now supported! (:issue:`622`) - Python 3.5 is no longer supported. @@ -2090,7 +2287,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you 0.29.2 (2019-07-13) ------------------- -- Bumped `Uvicorn `__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`) +- Bumped `Uvicorn `__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`) - Fixed bug where the navigation breadcrumbs were not displayed correctly on the page for a custom query. (:issue:`558`) - Fixed bug where custom query names containing unicode characters caused errors. @@ -2112,7 +2309,7 @@ ASGI, new plugin hooks, facet by date and much, much more... ASGI ~~~~ -`ASGI `__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 `__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn `__ and no longer depends on `Sanic `__. +`ASGI `__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 `__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn `__ and no longer depends on `Sanic `__. I wrote about the significance of this change in `Porting Datasette to ASGI, and Turtles all the way down `__. @@ -2526,7 +2723,7 @@ Miscellaneous as a string. * If you just want an array of the first value of each row, use the new ``?_shape=arrayfirst`` option - `example - `_. + `_. 0.22.1 (2018-05-23) ------------------- diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7ca88c4e..db903e8e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -76,6 +76,8 @@ Or since this is the default command you can run this instead:: Once started you can access it at ``http://localhost:8001`` +Use ``--internal PATH`` or the ``DATASETTE_INTERNAL`` environment variable to persist :ref:`Datasette's internal database ` to a SQLite file. + .. [[[cog help(["serve", "--help"]) .. ]]] @@ -244,6 +246,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam custom query (default=1000) max_insert_rows Maximum rows that can be inserted at a time using the bulk insert API (default=100) + max_post_body_bytes Maximum size in bytes for a POST body read into + memory, e.g. JSON API requests - set 0 to disable + this limit (default=2097152) num_sql_threads Number of threads in the thread pool for executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds diff --git a/docs/conf.py b/docs/conf.py index 5dd06b57..2a5c1439 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Datasette documentation build configuration file, created by # sphinx-quickstart on Thu Nov 16 06:50:13 2017. diff --git a/docs/contributing.rst b/docs/contributing.rst index 692f94c8..2de1fce0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -312,6 +312,19 @@ To update these pages, run the following command:: uv run cog -r docs/*.rst +.. _contributing_documentation_screenshots: + +Documentation screenshots +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Screenshots in the documentation are defined in ``docs/shots.yml`` and taken using `shot-scraper `__. That file starts a Datasette server that loads JavaScript from ``docs/shots/``, then saves each screenshot as a WebP image in ``docs/images/``. + +To take any screenshots that do not exist yet, run:: + + just shots + +``just docs`` runs this too. Existing images are skipped. To replace a screenshot, delete its image file and run ``just shots`` again. + .. _contributing_template_contexts: Documented template contexts diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 50c23b3e..edb70670 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -302,6 +302,14 @@ content you can do so by creating a ``row.html`` template like this: Note the ``default:row.html`` template name, which ensures Jinja will inherit from the default template. +The default ``base.html`` template provides a ``crumbs`` block inside its navigation block. Override ``crumbs`` to customize the breadcrumbs without replacing the rest of the navigation. The imported ``crumbs.nav()`` macro renders Datasette's permission-aware breadcrumbs: + +.. code-block:: jinja + + {% block crumbs %} + {{ crumbs.nav(request=request, database=database, table=table) }} + {% endblock %} + The ``_table.html`` template is included by both the row and the table pages, and a list of rows. The default ``_table.html`` template renders them as an HTML template and `can be seen here `_. diff --git a/docs/facets.rst b/docs/facets.rst index 960ac03a..6bb822e0 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -7,13 +7,15 @@ Datasette facets can be used to add a faceted browse interface to any database t With facets, tables are displayed along with a summary showing the most common values in specified columns. These values can be selected to further filter the table. -Here's `an example `__: +Here's `an example `__: .. image:: https://raw.githubusercontent.com/simonw/datasette-screenshots/0.62/non-retina/faceting-details.png :alt: Screenshot showing facets against a table of congressional legislators. Suggested facets include state_rank and start and end dates, and the displayed facets are state, party and type. Each facet lists values along with a count of rows for each value. Facets can be specified in two ways: using query string parameters, or in ``metadata.json`` configuration for the table. +Facet queries are generated by Datasette and summarize rows the actor already has permission to view. They do not require the :ref:`actions_execute_sql` permission. Use the :ref:`setting_allow_facet` setting to control whether users can request facets using query string parameters. + Facets in query strings ----------------------- diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index 349ad149..8b627cdc 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -19,7 +19,7 @@ The table page and table view API Table views that support full-text search can be queried using the ``?_search=TERMS`` query string parameter. This will run the search against content from all of the columns that have been included in the index. -Try this example: `fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort `__ +Try `searching Datasette ecosystem repositories for "csv" `__ to find tools for working with CSV files. SQLite full-text search supports wildcards. This means you can easily implement prefix auto-complete by including an asterisk at the end of the search term - for example:: @@ -52,11 +52,11 @@ Configuring full-text search for a table or view If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table. -You can also manually configure which table should be used for full-text search using query string parameters or table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option. +You can also manually configure which table should be used for full-text search using table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option. -Use ``?_fts_table=x`` to over-ride the FTS table for a specific page. If the primary key was something other than ``rowid`` you can use ``?_fts_pk=col`` to set that as well. This is particularly useful for views, for example: +The legacy ``?_fts_table=x`` and ``?_fts_pk=col`` query string parameters are accepted only if they exactly match the configured or automatically detected FTS mapping. They cannot be used to select a different FTS table or primary key. This prevents a public table from being used to probe the contents of a private FTS table. -https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk +Searching also requires the current actor to have ``view-table`` permission for the FTS table itself, in addition to permission to view the table or view being searched. The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property. @@ -120,41 +120,31 @@ Searches using custom SQL You can include full-text search results in custom SQL queries. The general pattern with SQLite search is to run the search as a sub-select that returns rowid values, then include those rowids in another part of the query. -You can see the syntax for a basic search by running that search on a table page and then clicking "View and edit SQL" to see the underlying SQL. For example, consider this search for `manafort is the US FARA database `_:: +You can see the syntax for a basic search by running that search on a table page and then clicking "View and edit SQL" to see the underlying SQL. For example, consider this search for `repositories mentioning "csv" `_:: - /fara/FARA_All_ShortForms?_search=manafort + /content/repos?_search=csv -If you click `View and edit SQL `_ you'll see that the underlying SQL looks like this: +The generated SQL selects all columns in the table. This simplified version selects just the repository ID, full name and description, using the same full-text search condition. `Run this query `_ with ``search`` set to ``csv``: .. code-block:: sql select - rowid, - Short_Form_Termination_Date, - Short_Form_Date, - Short_Form_Last_Name, - Short_Form_First_Name, - Registration_Number, - Registration_Date, - Registrant_Name, - Address_1, - Address_2, - City, - State, - Zip + id, + full_name, + description from - FARA_All_ShortForms + repos where rowid in ( select rowid from - FARA_All_ShortForms_fts + repos_fts where - FARA_All_ShortForms_fts match escape_fts(:search) + repos_fts match escape_fts(:search) ) order by - rowid + id limit 101 diff --git a/docs/images/modal-classes.webp b/docs/images/modal-classes.webp new file mode 100644 index 00000000..9de7576f Binary files /dev/null and b/docs/images/modal-classes.webp differ diff --git a/docs/images/modal-example.webp b/docs/images/modal-example.webp new file mode 100644 index 00000000..6ba95686 Binary files /dev/null and b/docs/images/modal-example.webp differ diff --git a/docs/index.rst b/docs/index.rst index d494fd17..bf109f68 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Contents javascript_plugins plugin_hooks testing_plugins + plugin_telemetry internals events upgrade_guide diff --git a/docs/installation.rst b/docs/installation.rst index 33d3d6a1..ceec7f23 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -17,13 +17,6 @@ If you want to start making contributions to the Datasette project by installing Basic installation ================== -.. _installation_datasette_desktop: - -Datasette Desktop for Mac -------------------------- - -`Datasette Desktop `__ is a packaged Mac application which bundles Datasette together with Python and allows you to install and run Datasette directly on your laptop. This is the best option for local installation if you are not comfortable using the command line. - .. _installation_homebrew: Using Homebrew diff --git a/docs/internals.rst b/docs/internals.rst index c826de1a..6ecf5b73 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -26,7 +26,7 @@ The request object is passed to various plugin hooks. It represents an incoming The request scheme - usually ``https`` or ``http``. ``.headers`` - dictionary (str -> str) - A dictionary of incoming HTTP request headers. Header names have been converted to lowercase. + A dictionary of incoming HTTP request headers. Header lookups using ``request.headers["Content-Type"]``, ``request.headers.get("Content-Type")`` and ``"Content-Type" in request.headers`` are case-insensitive. Header names are lowercase when iterating over the dictionary. ``.cookies`` - dictionary (str -> str) A dictionary of incoming cookies @@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming ``.actor`` - dictionary (str -> Any) or None The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated. +``.max_post_body_bytes`` - integer + The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument. + The object also has the following awaitable methods: ``await request.form(files=False, ...)`` - FormData @@ -109,9 +112,11 @@ The object also has the following awaitable methods: ``await request.json()`` - Any Returns the parsed JSON body of a request submitted by ``POST``. -``await request.post_body()`` - bytes +``await request.post_body(max_bytes=None)`` - bytes Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data. + The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them. + And a class method that can be used to create fake request objects for use in tests: ``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)`` @@ -142,7 +147,7 @@ And a class method that can be used to create fake request objects for use in te .. _internals_multiparams: The MultiParams class -===================== +--------------------- ``request.args`` is a ``MultiParams`` object - a dictionary-like object which provides access to query string parameters that may have multiple values. @@ -172,7 +177,7 @@ Consider the query string ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` a .. _internals_formdata: The FormData class -================== +------------------ ``await request.form()`` returns a ``FormData`` object - a dictionary-like object which provides access to form fields and uploaded files. It has a similar interface to ``MultiParams``. @@ -200,7 +205,7 @@ The FormData class .. _internals_uploadedfile: The UploadedFile class -====================== +---------------------- When parsing multipart form data with ``files=True``, file uploads are returned as ``UploadedFile`` objects with the following properties and methods: @@ -279,7 +284,7 @@ For example: content_type="application/xml; charset=utf-8", ) -The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods: +The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)``, ``Response.error(...)`` or ``Response.redirect(...)`` helper methods: .. code-block:: python @@ -290,6 +295,8 @@ The quickest way to create responses is using the ``Response.text(...)``, ``Resp text_response = Response.text( "This will become utf-8 encoded text" ) + # A JSON error in Datasette's standard error format: + error_response = Response.error("Cannot do that", 400) # Redirects are served as 302, unless you pass status=301: redirect_response = Response.redirect( "https://latest.datasette.io/" @@ -299,6 +306,8 @@ Each of these responses will use the correct corresponding content-type - ``text Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above. +``Response.error(messages, status=400)`` returns a JSON error in the :ref:`standard Datasette error format `. ``messages`` can be a single string or a list of strings. Use this for JSON-only endpoints; if your error should content-negotiate between JSON and HTML, raise ``Forbidden``, ``NotFound``, ``BadRequest`` or ``DatasetteError`` instead and Datasette's error handling will build the appropriate response. + .. _internals_response_asgi_send: Returning a response with .asgi_send(send) @@ -1394,7 +1403,31 @@ Release all resources held by this ``Datasette`` instance. This calls :ref:`data If a call to ``Database.close()`` on one of the attached databases raises an exception, ``Datasette.close()`` will continue trying to close the remaining databases and will re-raise the first exception after every database has been processed. -When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. +When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. See :ref:`datasette_lifecycle` for where ``close()`` fits into the full startup-to-shutdown sequence. + +.. _datasette_add_background_task: + +.add_background_task(func, name=None) +------------------------------------- + +``func`` - async callable + A coroutine function taking one positional argument, the ``Datasette`` instance. Core calls ``await func(datasette)``. + +``name`` - string, optional + A name for the task, used to identify it in the ``/-/tasks`` introspection endpoint (:ref:`JsonDataView_tasks`) and in log messages. Defaults to ``func.__qualname__``. If the resulting name collides with an already-registered task, a ``-2``, ``-3``, ... suffix is appended. + +Registers supervised background work and returns a :ref:`BackgroundTask ` handle. Tasks registered during startup launch after all startup hooks finish; tasks registered after launch start immediately. + +See :ref:`internals_background_tasks` for examples, launch behavior, task supervision and cancellation. + +.. _datasette_start_background_tasks: + +await .start_background_tasks() +------------------------------- + +Runs startup (if it has not already run) and launches every task registered with :ref:`datasette_add_background_task`. + +See :ref:`internals_background_tasks` for when tasks launch automatically, and :ref:`internals_background_tasks_explicit` for examples and startup considerations in tests and headless programs. .. _datasette_track_event: @@ -1585,32 +1618,32 @@ datasette.client Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs. -The ``datasette.client`` object is a wrapper around the `HTTPX Python library `__, providing an async-friendly API that is similar to the widely used `Requests library `__. +The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library `__, providing an async-friendly API that is similar to the widely used `Requests library `__. It offers the following methods: -``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response Execute an internal GET request against that path. -``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters. -``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response Execute an internal OPTIONS request. -``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response Execute an internal HEAD request. -``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response Execute an internal PUT request. -``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response Execute an internal PATCH request. -``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response +``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response Execute an internal DELETE request. -``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response +``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response Execute an internal request with the given HTTP method against that path. These methods can be used with :ref:`internals_datasette_urls` - for example: @@ -1627,7 +1660,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example: ``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path. -For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation `__. +For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation `__. .. _internals_datasette_client_actor: @@ -1756,6 +1789,135 @@ Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) a These methods each return a ``datasette.utils.PrefixedUrlString`` object, which is a subclass of the Python ``str`` type. This allows the logic that considers the ``base_url`` setting to detect if that prefix has already been applied to the path. +.. _datasette_lifecycle: + +Application lifecycle +===================== + +Datasette guarantees a fixed sequence of events between the moment a ``Datasette`` instance is constructed and the moment its resources are released: + +1. ``Datasette(...)`` — the constructor runs synchronously and does not run plugin hooks. +2. **Startup** — ``await datasette.invoke_startup()`` runs once: it populates the internal database's catalog of table schemas (:ref:`internals_internal`), loads canned queries and column type configuration, then calls every registered :ref:`plugin_hook_startup` hook, in plugin registration order. When Datasette is being served, table-count precomputation for immutable databases runs immediately before this, as part of the same startup sequence. +3. **Background-task launch** — once *every* ``startup`` hook has finished (not before), every task registered with :ref:`datasette_add_background_task` — by any plugin — is launched. A task registered by one plugin's ``startup`` hook can safely depend on state set up by another plugin's ``startup`` hook, because launch only happens after the whole round of hooks completes. +4. **Serving** — the instance handles requests (or, for headless or CLI use, does whatever the embedding program does with it). +5. **Shutdown** — triggered by the ASGI ``lifespan.shutdown`` event (Ctrl-C, ``SIGTERM``) or the end of a ``datasette serve`` process: every :ref:`plugin_hook_shutdown` hook runs first, while background tasks are still alive, so a plugin can tell its own task to wind down gracefully; every still-running background task is then cancelled and given a five-second grace period to actually stop; finally every database connection is released via :ref:`datasette_close`. + +.. admonition:: Startup hooks run on the event loop that serves requests + + In every trigger path below, ``startup`` hooks run on the same ``asyncio`` event loop that goes on to accept connections. It is safe to create loop-bound primitives — ``asyncio.Lock``, ``asyncio.Queue``, ``asyncio.Event``, a raw ``asyncio.create_task()`` call — inside a ``startup`` hook, and to register long-lived background work with :ref:`datasette_add_background_task` there. + +Three trigger paths +------------------- + +- ``datasette serve`` (CLI) — startup and ``uvicorn.Server.serve()`` both run inside a single ``asyncio.run()`` call, so there is exactly one event loop for the whole life of the process. +- **ASGI lifespan** — ``Datasette.app()`` wires startup and background-task launch into the ``on_startup`` list, and shutdown into the ``on_shutdown`` list, of an internal ``AsgiLifespan`` wrapper. A spec-compliant ASGI server (uvicorn, hypercorn, and others) sends the ``lifespan.startup`` message and waits for ``lifespan.startup.complete`` before delivering any ``http`` or ``websocket`` scope, so startup — including every plugin's own internal-database migrations — is guaranteed to have finished before any request reaches Datasette, including requests seen by plugin :ref:`asgi_wrapper ` middleware. If a ``startup`` hook raises, ``AsgiLifespan`` sends ``lifespan.startup.failed`` with the exception message instead of hanging or crashing ambiguously, so the host can abort the boot cleanly. +- **First-request fallback** — an internal ``AsgiRunOnFirstRequest`` wrapper runs the same startup work as a safety net for hosts that never send ASGI lifespan events at all: some ASGI mounts, a bare ``app()`` embedded inside another framework, and :ref:`datasette.client ` / test clients, which drive requests directly over ``httpx2.ASGITransport`` without ever emitting ``lifespan.startup``. It runs startup exactly once, the first time any non-lifespan scope arrives, guarded by a lock so that concurrent early requests can't run it twice. + +All three paths call the same idempotent internal methods, so it is safe for more than one of them to fire — lifespan startup completing and then a first request arriving afterwards is a no-op the second time. A host that never sends lifespan events and never goes through the CLI degrades to first-request timing: startup runs on the first request instead of before it, exactly as Datasette always worked prior to this lifecycle guarantee. This is a deliberate fallback rather than a regression — see :ref:`internals_background_tasks` for how to opt out of launching background tasks (the ``--get`` CLI path) or drive startup and launch explicitly (tests, headless embedders). + +.. _internals_background_tasks: + +Background tasks +================ + +Datasette can supervise long-lived background work for plugins, such as polling for updates. Register work using :ref:`datasette_add_background_task` and use the returned :ref:`BackgroundTask ` handle to inspect or cancel it. See :ref:`datasette_lifecycle` for how background tasks fit into the application's startup and shutdown sequence. + +Registering tasks +----------------- + +Use :ref:`datasette_add_background_task` to register an async callable, typically from a :ref:`plugin_hook_startup` hook. The callable takes one argument, the ``Datasette`` instance. Tasks can also be registered later, including from a request handler. + +Registration is separate from launch. Calling this from a ``startup`` hook — the common case — buffers the task; core launches every registered task once *all* ``startup`` hooks have completed, as described in :ref:`datasette_lifecycle`. Calling it after launch has already happened — for example from a request handler, to start a per-job task dynamically — starts the task immediately instead. + +.. code-block:: python + + import asyncio + from datasette import hookimpl + + + async def poll_for_updates(datasette): + while True: + await do_one_poll(datasette) + await asyncio.sleep(60) + + + @hookimpl + def startup(datasette): + datasette.add_background_task( + poll_for_updates, name="my-plugin-poller" + ) + +Datasette supervises each registered task: + +- Keeps the task alive. +- Logs exceptions other than ``asyncio.CancelledError``, with their tracebacks, to the ``datasette.background_tasks`` logger. The exception is recorded on the handle's ``.exception``, and its ``.state`` becomes ``crashed``. +- Cancels running tasks during shutdown and gives them five seconds to stop. See :ref:`datasette_lifecycle`. + +.. _internals_background_tasks_launch: + +Launch matrix +------------- + +Whether registered tasks actually launch depends on how the instance is being run: + +.. list-table:: + :header-rows: 1 + + * - Trigger + - Launches registered tasks? + * - ASGI lifespan (real server deployments) + - Yes, after ``lifespan.startup`` completes + * - First-request fallback (lifespan-less hosts) + - Yes, on the first request — parity with the lifespan case + * - ``datasette serve --get`` + - Never + * - Tests / headless embedders + - Only if you call :ref:`datasette_start_background_tasks` explicitly + +``datasette --get`` never launches background tasks, even though its one-shot request flows through the same first-request fallback as everything else: it sets an internal flag before making that request specifically to suppress the launch, since a one-shot CLI invocation has no server loop left running afterwards to keep any launched tasks alive. + +.. _internals_background_tasks_explicit: + +Starting tasks explicitly +------------------------- + +Call :ref:`datasette_start_background_tasks` to run startup (if it has not already run) and launch every task registered with :ref:`datasette_add_background_task`. This is the explicit equivalent of what happens automatically via ASGI lifespan or the first-request fallback in a served deployment — the entry point for tests and headless embedders (a cron-style CLI command that wants supervised background work without running a server) that need background tasks without going through either of those paths. + +.. code-block:: python + + datasette = Datasette(memory=True) + await datasette.start_background_tasks() + +.. _BackgroundTask: + +BackgroundTask objects +---------------------- + +:ref:`datasette_add_background_task` returns a ``BackgroundTask`` handle with the following attributes: + +``.name`` - string + The task's (unique) name. + +``.state`` - string + One of ``registered`` (added but not yet launched), ``running``, ``completed`` (returned cleanly), ``crashed`` (raised an exception) or ``cancelled``. + +``.task`` - ``asyncio.Task`` or ``None`` + The underlying ``asyncio.Task``, once launched. ``None`` while still ``registered``. + +``.exception`` - ``BaseException`` or ``None`` + The exception that crashed the task, if ``.state`` is ``crashed``. + +``.started_at`` - string or ``None`` + ISO 8601 UTC timestamp of when the task was launched. + +``.function`` - string + The callable's dotted module and qualified name, for example ``my_plugin.jobs.poll_for_updates``. + +``.cancel()`` + Cancel the task. If it has already launched, this cancels the underlying ``asyncio.Task`` — ``.state`` becomes ``cancelled`` once the cancellation is observed. If it has not launched yet, it is removed from the queue so it never runs. + +This is also the shape of each entry returned by the ``/-/tasks`` JSON introspection endpoint — see :ref:`JsonDataView_tasks`. + .. _internals_permission_classes: Permission classes and utilities @@ -2012,10 +2174,12 @@ Example usage: version = await db.execute_fn(get_version) +The call is traced as a ``db.query`` OpenTelemetry span carrying ``datasette.callback`` (the function's qualified name) rather than ``db.query.text``, since the SQL is whatever the function chooses to run - see :ref:`internals_telemetry`. Passing a named function gives the span a readable identity; a lambda reports ````. + .. _database_execute_write: -await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10) --------------------------------------------------------------------------------------------------------- +await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True, time_limit_ms=2000) +---------------------------------------------------------------------------------------------------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -2050,7 +2214,16 @@ If you need to retrieve every row returned by a statement, pass ``return_all=Tru If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. -Each call to ``execute_write()`` will be executed inside a transaction. +Each call to ``execute_write()`` will be executed inside a transaction. Pass +``transaction=False`` for statements such as ``VACUUM`` that cannot run inside +a transaction. + +Write statements have a default time limit of 2,000ms. Pass a different value +using ``time_limit_ms=`` or use ``time_limit_ms=None`` to allow the statement to +run without a time limit. + +This write limit is independent of the ``sql_time_limit_ms`` setting used for +read queries. Changing that setting does not change the default write limit. .. _database_execute_write_script: @@ -2086,6 +2259,8 @@ This method works like ``.execute_write()``, but instead of a SQL statement you The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection for as long as it is executing. +Like ``execute_fn()``, the call is traced as a ``db.query`` OpenTelemetry span carrying ``datasette.callback`` rather than ``db.query.text``, above the write-queue spans - see :ref:`internals_telemetry`. A named function gives the span a readable identity; a lambda reports ````. + .. warning:: ``fn`` needs to be a regular function, not an ``async def`` function. @@ -2142,7 +2317,25 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. -By default your function will be executed inside a transaction. You can pass ``transaction=False`` to disable this behavior, though if you do that you should be careful to manually apply transactions - ideally using the ``with conn:`` pattern, or you may see ``OperationalError: database table is locked`` errors. +By default Datasette manages the transaction. For nested transactions, use `sqlite_utils.Database(conn).atomic() `__. Pass ``transaction=False`` to manage transactions yourself. + +For example, archive an article and record the change in an audit log: + +.. code-block:: python + + import sqlite_utils + + + def archive_article(conn): + db = sqlite_utils.Database(conn) + with db.atomic(): + db["articles"].update(1, {"archived": True}) + db["audit_log"].insert( + {"article_id": 1, "action": "archive"} + ) + + + await database.execute_write_fn(archive_article) If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. @@ -2302,6 +2495,259 @@ The ``Database`` class also provides properties and methods for introspecting th } } +.. _internals_telemetry: + +OpenTelemetry +============= + +Datasette uses the `opentelemetry-api `__ library to provide `OpenTelemetry `__ traces and metrics for Datasette applications. + +Datasette emits telemetry under the ``datasette`` instrumentation scope. To enable tracing, run Datasette under the ``opentelemetry-instrument`` agent. + +Plugins can emit their own spans and metrics alongside these, using the same registry classes and test helpers core uses - see :ref:`plugin_telemetry`. + +.. _internals_telemetry_turning_on: + +Turning tracing on +------------------ + +Install an OpenTelemetry SDK, an exporter and the instrumentation agent, then launch Datasette through ``opentelemetry-instrument``: + +.. code-block:: bash + + pip install opentelemetry-distro opentelemetry-exporter-otlp + + OTEL_SERVICE_NAME=datasette \ + OTEL_METRICS_EXPORTER=console \ + OTEL_LOGS_EXPORTER=console \ + OTEL_TRACES_EXPORTER=console \ + opentelemetry-instrument datasette mydb.db + +Or using ``uv run``: + +.. code-block:: bash + + OTEL_SERVICE_NAME=datasette \ + OTEL_METRICS_EXPORTER=console \ + OTEL_LOGS_EXPORTER=console \ + OTEL_TRACES_EXPORTER=console \ + uv run \ + --with opentelemetry-distro \ + --with opentelemetry-exporter-otlp \ + opentelemetry-instrument datasette mydb.db + +This will output pretty-printed JSON telemetry to your console, representing requests and database queries executed by Datasette. + +To use an exporter endpoint, set ``OTEL_EXPORTER_OTLP_ENDPOINT`` to a URL, set ``OTEL_TRACES_EXPORTER`` to ``otlp``, and set the other exporters to ``none``: + +.. code-block:: bash + + OTEL_SERVICE_NAME=datasette \ + OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ + OTEL_METRICS_EXPORTER=none \ + OTEL_LOGS_EXPORTER=none \ + OTEL_TRACES_EXPORTER=otlp \ + opentelemetry-instrument datasette mydb.db + +On macOS one easy option for a port 4317 OTLP endpoint is `otel-tui `__: + +.. code-block:: bash + + brew install ymtdzzz/tap/otel-tui + otel-tui + +Traces sent to port 4317 by Datasette will now display in a TUI in your terminal. + +A few things catch people out: + +- **You must use "opentelemetry-instrument datasette"**. Running just ``OTEL_TRACES_EXPORTER=console datasette mydb.db`` produces no telemetry. +- **Spans do not appear immediately.** The SDK's default ``BatchSpanProcessor`` flushes on a timer, every 5 seconds. Either wait, or stop the process - shutdown triggers a final flush - or set ``OTEL_BSP_SCHEDULE_DELAY=1000`` while you are experimenting. +- **Always set** ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for. + +.. _internals_telemetry_requests: + +Span reference +-------------- + +Datasette emits six spans. One covers the HTTP request, and is the root everything else raised while serving that request hangs from. Four describe the database layer - one per query, one for the work that query does inside a SQL worker thread, and two more for the write queue. The sixth covers startup. Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``. + +A request to a table page produces a span named, in full:: + + GET /(?P[^\/\.]+)/(?P
      [^\/\.]+)(\.(?P\w+))?$ + +.. [[[cog + from telemetry_doc import spans + spans(cog) +.. ]]] + +``{http.request.method} {http.route}`` + One span per HTTP request, containing spans from plugin middleware and database operations. Named for the HTTP method and matched route, or just the method if no route matches. Incoming ``traceparent`` headers are extracted using the global propagator to continue the caller's trace. Incoming ``baggage`` is not propagated into plugin or downstream context in this release. Set ``OTEL_PROPAGATORS=none`` to disable extraction. For public instances, strip trace context headers at your proxy if callers should not supply trace context. + + Kind: ``SERVER``. + + Attributes: + + - ``http.request.method`` - The HTTP request method. Methods outside the nine defined by RFC 9110 and RFC 5789 are recorded as ``_OTHER``. + - ``http.route`` *(optional)* - The regular expression for the matched route, for example ``/(?P[^\/\.]+)/(?P
      [^\/\.]+)(\.(?P\w+))?$`` for a table page. Use this attribute to group requests by route. Omitted when no route matches. + - ``url.path`` - The URL path, excluding the query string. + - ``url.scheme`` - ``http`` or ``https``. + - ``server.address`` *(optional)* - The ``Host`` header, including any ``:port`` suffix. This value is supplied by the client. + - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none. + - ``http.response.status_code`` *(optional)* - The HTTP response status code. Omitted if no response was started. + - ``error.type`` *(optional)* - The exception class name for a failed operation. On HTTP spans, also set to the status code as a string for 5xx responses. A 4xx response alone does not set this attribute or an error status. + - ``datasette.internal_client`` *(optional)* - ``True`` for requests made through ``datasette.client``. Calls made inside another request produce a nested ``SERVER`` span. Filter on this attribute to exclude internal requests from request counts. Omitted for requests received over the network. + +``db.query`` + A SQL operation, including time spent queued for a worker thread. For ``block=False`` writes, the span ends after the write is queued. Callback methods record ``datasette.callback`` in place of ``db.query.text``. + + Kind: ``CLIENT``. + + Attributes: + + - ``db.system`` - Always ``sqlite``. + - ``db.namespace`` - Name of the database being queried. + - ``db.query.text`` *(optional)* - The SQL, truncated to 2048 characters. Bound parameter values are not recorded. For callback methods, ``datasette.callback`` is recorded instead. + - ``datasette.callback`` *(optional)* - The qualified name of the Python callable passed to ``execute_fn()``, ``execute_write_fn()`` or ``execute_isolated_fn()``, for example ``TableInsertView.post..insert_or_upsert_rows``. Set instead of ``db.query.text``. Lambdas appear as ````; use a named function for a more descriptive span. + - ``db.operation.name`` *(optional)* - The statement's leading keyword, such as ``SELECT``, ``INSERT`` or ``CREATE``, if it matches the supported allowlist. Statements beginning with a common table expression report ``WITH``. Omitted for unrecognized keywords and ``execute_write_script()``. + - ``datasette.param_count`` *(optional)* - Number of bound parameters. Recorded instead of the values themselves. + - ``datasette.param_sets`` *(optional)* - Number of parameter sets consumed by ``execute_write_many()``. The parameter values are not recorded. + - ``datasette.time_limit_ms`` *(optional)* - Time limit applied to the read query, in milliseconds: :ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``. + - ``datasette.rows_returned`` *(optional)* - Number of rows returned by a successful read query. + - ``datasette.truncated`` *(optional)* - True if the result was cut short by :ref:`setting_max_returned_rows`. + - ``datasette.interrupted`` *(optional)* - True if the query exceeded its time limit. The span status is set to ``ERROR`` unless the caller used a ``custom_time_limit`` shorter than :ref:`setting_sql_time_limit_ms`, in which case the status is left unset. + - ``datasette.sql_error_suppressed`` *(optional)* - True for a non-timeout SQL error with ``log_sql_errors=False``. The exception is still raised, but the span status is left unset. + - ``datasette.executescript`` *(optional)* - True for ``execute_write_script()``, which runs multiple statements. + - ``datasette.executemany`` *(optional)* - True for ``execute_write_many()``, which runs one statement against many parameter sets. + +``db.query.execute`` + The read executing inside a SQL worker thread. Child of ``db.query``; the gap between the two is time spent waiting for a thread. + + No attributes. + +``db.write.queue_wait`` + Time a write spent waiting in its database's write queue. For ``block=True``, this is a child of ``db.query``. For ``block=False``, it is a root span linked to the span that queued the write, since the write can outlive that request. + + No attributes. + +``db.write.execute`` + The write executing on the write thread. For ``block=True``, this is a child of ``db.query``. For ``block=False``, it is a root span linked to the span that queued the write. + + Attributes: + + - ``datasette.isolated_connection`` - True if the write ran on its own connection rather than the shared write connection. + - ``datasette.transaction`` - False for statements such as ``VACUUM`` that cannot run inside a transaction. + +``datasette.startup`` + Startup work performed by ``invoke_startup()``, including registration hooks, schema catalog updates, saved queries, column type configuration and the ``startup`` hook. Runs during instance startup, either before serving requests or as part of the first request. + + No attributes. + +.. [[[end]]] + +.. _internals_telemetry_metrics: + +Metric reference +---------------- + +Spans describe events; metrics describe levels and rates. Metrics can be used to answer questions like "Am I saturating my :ref:`setting_num_sql_threads` threads right now?". Trace sampling drops a portion of traces but does not drop any metrics. + +Datasette configures duration histograms in **seconds**. OpenTelemetry's default boundaries are tuned for milliseconds but these would file every SQLite query into a single bucket, making quantile queries meaningless. + +This reference is also generated from ``datasette/telemetry_registry.py``: + +.. [[[cog + from telemetry_doc import metrics + metrics(cog) +.. ]]] + +``db.client.operation.duration`` + Histogram, unit ``s``. Duration of a SQL operation, including callback-based calls such as ``execute_fn()``. For ``block=False`` writes, measures enqueue time. + + Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``. + + Attributes: + + - ``db.system`` - Always ``sqlite``. + - ``db.namespace`` - Name of the database being queried. + - ``datasette.operation`` - Whether the operation was a read or a write. One of: ``read``, ``write``. + - ``error.type`` *(optional)* - The exception class name for a failed operation. On HTTP spans, also set to the status code as a string for 5xx responses. A 4xx response alone does not set this attribute or an error status. + +``datasette.write.queue_wait`` + Histogram, unit ``s``. Time each write waited in its database's write queue. + + Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.sql.queries.interrupted`` + Counter, unit ``{query}``. Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A rising rate can indicate that queries need optimization or a higher time limit. Caller-selected timeouts shorter than this limit, such as those used for facet suggestion, are excluded. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.sql.threads.limit`` + Observable gauge, unit ``{thread}``. Maximum concurrent read queries, configured by :ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` is ``0``. + + No attributes. + +``datasette.sql.threads.queue_depth`` + Observable gauge, unit ``{query}``. Read queries waiting for a free SQL thread. Sustained values above zero indicate a saturated read pool. + + No attributes. + +``datasette.sql.queries.pending`` + Observable gauge, unit ``{query}``. Read queries submitted to the pool and not yet complete. Sum across databases and compare with ``datasette.sql.threads.limit`` to assess pool usage. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.write.queue_depth`` + Observable gauge, unit ``{write}``. Writes waiting for a database's single write thread. Increasing ``num_sql_threads`` does not increase write concurrency. Not reported for databases that have never been written to. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.connections.open`` + Observable gauge, unit ``{connection}``. Open SQLite connections managed by Datasette. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +.. [[[end]]] + +Exemplars +~~~~~~~~~ + +An OpenTelemetry `exemplar `__ attaches a trace ID and span ID to one sample backing a histogram measurement. Where a spike in ``db.client.operation.duration`` alone tells you "queries were slow sometime in this minute", the exemplar attached to one of the samples in that spike gives you the trace ID of an actual slow query to open: + +.. code-block:: text + + db.client.operation.duration count=4 + exemplars: 4 + value=0.001564s trace_id=ddfaf45fd4e14913497d7efeac95f381 span_id=fd5792bdbb01e533 + value=0.006320s trace_id=34aea775ade11a3c5f716695731000fe span_id=25ed9e29dd84dbee + value=0.045253s trace_id=a65cb58d1460a179f0d04046ff51ed0d span_id=7f34d6378c85d062 + value=0.305240s trace_id=6089f4c515c221c0ca7bb53667b37ac8 span_id=0516f4a6641eaa0b + +.. _internals_telemetry_privacy: + +Privacy and safety +------------------ + +Datasette does not configure a telemetry exporter itself. If you enable one, traces may contain sensitive information: + +- **SQL text is truncated to 2048 characters.** Literal values in that text are retained. Bound SQL parameter values are not added as attributes; ``datasette.param_count`` records only their count. +- **Request spans include URL paths, host names and User-Agent headers.** Paths can include identifying values such as row primary keys. Core does not add actor identifiers, cookies, authorization headers, client IP addresses or a ``url.query`` attribute. +- **Exception messages and tracebacks may be recorded.** These can contain data from requests or database operations. + +Review what your application and plugins record before exporting telemetry to an external service. Restrict access to exported data and configure redaction or filtering where needed. + .. _internals_csrf: CSRF protection @@ -2323,7 +2769,20 @@ No token, cookie, or hidden form field is needed. Any ```` i Datasette's internal database ============================= -Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a ``--internal`` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. + +To persist internal data across Datasette instances, use the ``--internal`` option to specify the path to a SQLite database: + +.. code-block:: bash + + datasette mydatabase.db --internal internal.db + +You can also set the ``DATASETTE_INTERNAL`` environment variable to specify this path without passing ``--internal`` each time: + +.. code-block:: bash + + export DATASETTE_INTERNAL=/path/to/internal.db + datasette mydatabase.db Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. @@ -2354,6 +2813,14 @@ The internal database schema is as follows: .. code-block:: sql + CREATE TABLE "_sqlite_migrations" ( + "id" INTEGER PRIMARY KEY, + "migration_set" TEXT, + "name" TEXT, + "applied_at" TEXT + ); + CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name" + ON "_sqlite_migrations" ("migration_set", "name"); CREATE TABLE catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, @@ -2579,6 +3046,13 @@ Async version of :ref:`call_with_supported_arguments `. + +The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise `, with the exception of the debug endpoints ``/-/threads``, ``/-/tasks`` and ``/-/actions``, whose shapes may change in future releases. + .. _JsonDataView_metadata: /-/metadata @@ -37,6 +41,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example `_: +Shows a list of currently installed plugins and their versions. `Plugins example `_: .. code-block:: json @@ -97,6 +102,7 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. T .. code-block:: json { + "ok": true, "settings": { "template_debug": true, "trace_debug": true, @@ -129,20 +136,47 @@ Any keys that include the one of the following substrings in their names will be /-/databases ------------ -Shows currently attached databases. `Databases example `_: +Shows currently attached databases that the current actor is allowed to view, based on the ``view-database`` permission. `Databases example `_: .. code-block:: json - [ - { - "hash": null, - "is_memory": false, - "is_mutable": true, - "name": "fixtures", - "path": "fixtures.db", - "size": 225280 - } - ] + { + "ok": true, + "databases": [ + { + "hash": null, + "is_memory": false, + "is_mutable": true, + "name": "fixtures", + "path": "fixtures.db", + "size": 225280 + } + ] + } + +.. _JsonDataView_actions: + +/-/actions +---------- + +Shows all actions registered with the permission system, including those added by plugins. Requires the ``permissions-debug`` permission. + +.. code-block:: json + + { + "ok": true, + "actions": [ + { + "name": "view-instance", + "abbr": "vi", + "description": "View Datasette instance", + "takes_parent": false, + "takes_child": false, + "resource_class": null, + "also_requires": null + } + ] + } .. _JumpView: @@ -160,6 +194,7 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name. .. code-block:: json { + "ok": true, "matches": [ { "name": "fixtures", @@ -188,6 +223,7 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``: .. code-block:: json { + "ok": true, "matches": [ { "name": "fixtures: facetable", @@ -215,11 +251,12 @@ Without those query string arguments, the page lists up to five tables with dete /-/threads ---------- -Shows details of threads and ``asyncio`` tasks. `Threads example `_: +Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``permissions-debug`` permission, since it exposes runtime internals. `Threads example `_: .. code-block:: json { + "ok": true, "num_threads": 2, "threads": [ { @@ -241,6 +278,42 @@ Shows details of threads and ``asyncio`` tasks. `Threads example `; see also :ref:`BackgroundTask ` for what each field below means, and :ref:`datasette_lifecycle` for when tasks are launched. This endpoint requires the ``permissions-debug`` permission, since a crashed task's ``exception`` field can reveal internals such as file paths or query text: + +.. code-block:: json + + { + "ok": true, + "tasks": [ + { + "name": "my_plugin.poll_for_updates", + "state": "running", + "function": "my_plugin.poll_for_updates", + "started_at": "2026-07-30T12:00:00+00:00", + "exception": null + }, + { + "name": "my_plugin.broken_task", + "state": "crashed", + "function": "my_plugin.broken_task", + "started_at": "2026-07-30T12:00:00+00:00", + "exception": "ValueError('something went wrong')" + } + ], + "launched": true + } + +Each entry's ``function`` identifies the callable by its dotted module and qualified name. + +Each entry's ``state`` is one of ``registered`` (added but not yet launched), ``running``, ``completed``, ``crashed`` or ``cancelled``. ``exception`` is a one-line ``repr()`` of the exception for a ``crashed`` task, or ``null`` otherwise - the full traceback is written to the ``datasette.background_tasks`` logger instead, to keep this payload skimmable. + +The top-level ``launched`` flag reports whether the instance has run its one-time background task launch (after ``startup`` hooks finish, or via lifespan/first-request/:ref:`start_background_tasks() `). It distinguishes "no tasks have been registered" (``tasks`` is empty either way) from "tasks are registered but nothing has armed the launch yet" (``launched`` is ``false`` and every task's ``state`` is still ``registered``) - useful when debugging a host that never triggers Datasette's lifespan events. + .. _JsonDataView_actor: /-/actor @@ -251,6 +324,7 @@ Shows the currently authenticated actor. Useful for debugging Datasette authenti .. code-block:: json { + "ok": true, "actor": { "id": 1, "username": "some-user" diff --git a/docs/javascript_plugins.rst b/docs/javascript_plugins.rst index c4283cac..159f952d 100644 --- a/docs/javascript_plugins.rst +++ b/docs/javascript_plugins.rst @@ -1,7 +1,7 @@ .. _javascript_plugins: -JavaScript plugins -================== +JavaScript in plugins +===================== Datasette can run custom JavaScript in several different ways: @@ -35,7 +35,7 @@ Your JavaScript code can listen out for this event using ``document.addEventList datasetteManager ---------------- -The ``datasetteManager`` object +The ``datasetteManager`` object ``VERSION`` - string The version of Datasette @@ -474,6 +474,136 @@ Custom fields are responsible for preserving the accessibility of the form: Plugins should not submit the row themselves from inside ``makeColumnField()`` controls. Datasette owns the insert/edit dialog lifecycle, form submission, API call, error handling and row refresh. +.. _javascript_plugins_modals: + +Reusable modal dialogs +---------------------- + +Plugins can use ``DatasetteModal`` to create dialogs with the same appearance and keyboard behavior as Datasette's built-in dialogs. The component provides a native modal dialog, shared styles, Escape and backdrop dismissal, busy-state dismissal guards and focus restoration. + +Creating a dialog +~~~~~~~~~~~~~~~~~ + +``DatasetteModal.create()`` returns a detached ```` element containing a native ````. Access that native element through ``modal.dialog``. Populate its content before appending the wrapper to the page, then call ``modal.show()`` to open it. + +This example uses the :ref:`datasette_init event ` to add a button that opens a dialog: + +.. literalinclude:: shots/modal-example.js + :language: javascript + +Clicking that button opens this dialog: + +.. only:: not latex + + .. image:: images/modal-example.webp + :width: 584px + :alt: A dialog titled Example dialog, with the text "This dialog uses Datasette's shared styles and keyboard behavior." and a Close button in the footer, shown in front of a dimmed Datasette page + +Opening and closing +~~~~~~~~~~~~~~~~~~~ + +``modal.show(options)`` + Opens the native dialog using ``showModal()``. ``options`` is an optional object with these optional properties: + + - ``returnFocusTo`` (DOM element): Focus returns to this element when the dialog closes. Defaults to the element with keyboard focus immediately before the dialog opens. + - ``initialFocus`` (DOM element or function): An element inside the dialog whose ``focus()`` method will be called, or a function called with no arguments that moves focus itself. + + Use this to focus on an input field when the dialog opens. + +``modal.close(options)`` + Closes the dialog directly. ``options`` is an optional object with one optional property: + + - ``restoreFocus`` (boolean): Whether closing returns focus to the element recorded by ``show()``. Defaults to ``true``. + +``modal.requestClose(source)`` + Alternative to ``.close()`` that requests dismissal through the busy-state and ``beforeClose`` guards described below. Returns ``true`` if it closes the dialog, or ``false`` if the dialog is already closed or a guard prevents dismissal. Close and Cancel buttons should use this method. + + ``source`` is an optional string that is passed to ``beforeClose`` and identifies what requested dismissal. Datasette supplies ``"escape"`` for the Escape key or a native cancel event and ``"backdrop"`` for a click outside the dialog. ``source`` defaults to ``"cancel"``. + +Listen for the native dialog's ``close`` event to clean up resources such as pending requests or custom fields: + +.. code-block:: javascript + + modal.dialog.addEventListener("close", () => { + // Clean up content-specific resources here. + }); + +If the dialog is no longer needed, remove the wrapper with ``modal.remove()``. + +Dismissal guards and busy state +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can set ``modal.beforeClose`` to a synchronous function that receives the ``source`` string described above and returns ``false`` in order to keep the dialog open. + +Use ``source`` to decide what to do. This example prompts the user to ask if they want to discard unsaved changes - for example if they click outside the modal or hit Escape - but doesn't prompt them if they clicked a button like the Close one above that sets the ``source`` string to ``cancel``. + +.. code-block:: javascript + + modal.beforeClose = (source) => { + if (source === "cancel") return true; + return confirm("Discard unsaved changes?"); + }; + + +Set ``modal.busy = true`` while saving to prevent user dismissal. While busy, ``requestClose()`` returns ``false`` without calling ``beforeClose``. + +If an operation fails, set ``modal.busy = false`` so the user can retry or close the dialog. A successful operation can call ``modal.close()`` even while busy. + +.. _javascript_plugins_modal_classes: + +Shared CSS classes +~~~~~~~~~~~~~~~~~~ + +The classes in the example above provide built-in styling. This dialog uses every class listed below, including a ``modal-meta`` count in the header and ``footer-info`` text next to ``modal-btn-ghost`` and ``modal-btn-primary`` buttons in the footer: + +.. only:: not latex + + .. image:: images/modal-classes.webp + :width: 584px + :alt: A dialog titled Export rows with a "3 selected" badge in its header, a list of three plant names in the body, and a footer containing the text "CSV, UTF-8", a Cancel button and a blue Export button + +The following classes can be used by your modal: + +``datasette-modal`` + Added automatically to the native ```` when the wrapper is connected to the page. Provides the dialog's sizing, background, rounded corners, shadow, backdrop and animations. + +``modal-header`` + Adds padding, a bottom border and a horizontal layout for the title and optional metadata. + +``modal-title`` + Sets the title's font size, weight and color. Use ``aria-labelledby`` to associate the title with the dialog. + +``modal-meta`` + Styles optional metadata, such as a selected-item count, as small monospace text with a rounded background. + +``modal-body`` + Adds padding and makes overflowing content scroll while the header and footer remain visible. Sets ``min-height: 0``, ``overflow: auto`` and ``padding: 16px 24px 24px``. + +``modal-footer`` + Adds padding, a top border and a background to the action area. Arranges its contents horizontally, with buttons aligned to the right. + +``footer-info`` + Styles supporting text in the footer and lets it fill the space before the action buttons. + +``modal-btn`` + Provides base button styling, including padding, rounded corners, font and disabled appearance. Use it together with ``modal-btn-primary`` or ``modal-btn-ghost``. + +``modal-btn-primary`` + Gives a button an accent-colored background and white text, suitable for a primary action such as Save. + +``modal-btn-ghost`` + Gives a button a transparent background, muted text and a border, suitable for a secondary action such as Close or Cancel. + +These button classes are also used by Datasette's built-in dialogs. + +You can customize layout and sizing without adding extra classes. For example, this CSS uses the dialog's existing ID to widen it while keeping it inside the viewport: + +.. code-block:: css + + dialog#my-plugin-dialog { + width: min(720px, calc(100vw - 32px)); + } + .. _javascript_datasette_manager_selectors: Selectors diff --git a/docs/json_api.rst b/docs/json_api.rst index eca22fdc..73212e70 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -9,6 +9,54 @@ through the Datasette user interface can also be accessed as JSON via the API. To access the API for a page, either click on the ``.json`` link on that page or edit the URL and add a ``.json`` extension to it. +.. _json_api_stability: + +API stability +------------- + +Datasette 1.0 makes a stability promise for its JSON API: the endpoints, +parameters and response keys documented here and on the pages this +documentation links to will not change in backwards-incompatible ways for +the duration of the 1.x release series. + +Stability means: + +- Documented endpoints will keep their URLs, methods, parameters and + permission requirements. +- Documented response keys will keep their names and types. New keys may be + **added** in any release - clients should ignore keys they do not + recognize. +- The documented ``?_extra=`` names, ``?_shape=`` values and + :ref:`column filter operators ` are stable. +- Pagination tokens - the ``"next"`` key and ``?_next=`` parameter - are + **opaque strings**. Pass them back exactly as you received them; their + internal structure is not part of the API and can change at any time. +- The :ref:`standard error format ` and the + :ref:`API token format and restriction semantics ` are + stable, including the action abbreviations stored inside signed tokens. + +Some JSON endpoints are **exempt** from this promise: + +- Endpoints that are not documented include this marker key in their + responses and can change at any time:: + + "unstable": "This API is not part of Datasette's stable interface and may change at any time" + + This currently covers the instance homepage (``/.json``), the stored + query ``analyze``/``store``/``definition`` endpoints, ``/-/query/parameters``, + ``/-/execute-write/analyze`` and the JSON returned by the ``/-/permissions`` + debug playground. +- Debug and support endpoints are documented so you can use them, but their + JSON shapes are not frozen: :ref:`/-/threads `, + :ref:`/-/tasks `, + :ref:`/-/actions `, + the :ref:`permission debug endpoints ` + (``/-/allowed``, ``/-/rules``, ``/-/check``) and the + :ref:`table autocomplete endpoint `. +- Response keys explicitly labeled as unstable in this documentation, such + as the ``"analysis"`` block returned by :ref:`execute-write ` + and the ``debug`` and ``request`` extras. + .. _json_api_default: Default representation @@ -42,13 +90,49 @@ looks like this: "truncated": false } -``"ok"`` is always ``true`` if an error did not occur. +``"ok"`` is always ``true`` if an error did not occur. Every Datasette JSON endpoint that returns an object includes this key on success. The ``"rows"`` key is a list of objects, each one representing a row. The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). -For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. +For table pages, two additional keys are present: ``"next"``, an opaque token that can be used to retrieve the next page using ``?_next=TOKEN``, and ``"next_url"``, the full URL of that next page. Both are ``null`` on the final page. See :ref:`json_api_pagination`. + +.. _json_api_errors: + +Error responses +--------------- + +Every JSON error response from Datasette uses the same format: + +.. code-block:: json + + { + "ok": false, + "error": "Table not found", + "errors": [ + "Table not found" + ], + "status": 404 + } + +- ``"ok"`` is always ``false`` for an error. +- ``"errors"`` is a list of one or more error message strings. Endpoints that + validate multiple things at once - such as the :ref:`insert API ` - + may return several messages here. +- ``"error"`` is all of those messages joined with ``"; "``, for + convenience when displaying a single string. +- ``"status"`` matches the HTTP status code of the response. + +Some endpoints add extra context keys. For example, a SQL error from a +:ref:`custom query ` also includes the empty +``"rows"`` and ``"truncated"`` keys of the response it was unable to +produce. + +Permission errors use the same format: a request that fails a permission +check receives a ``403`` with this JSON error body when the URL ends in +``.json`` or the request sends an ``Accept: application/json`` or +``Content-Type: application/json`` header. .. _json_api_custom_sql: @@ -92,6 +176,7 @@ options: { "ok": true, "next": null, + "next_url": null, "rows": [ [3, "Detroit"], [2, "Los Angeles"], @@ -192,6 +277,10 @@ Here is an example Python function built using `requests `, for example ``{"ok": false, "error": "Unknown _extra: nope", ...}``. .. [[[cog from json_api_doc import table_extras @@ -266,6 +357,15 @@ The available table extras are listed below. 15 +``count_truncated`` + True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count. (May execute additional queries.) + + ``GET /fixtures/facetable.json?_extra=count,count_truncated`` + + .. code-block:: json + + false + ``count_sql`` SQL query string used to calculate the total count for the current table view, including active filters. @@ -338,17 +438,6 @@ The available table extras are listed below. "where state = \"CA\" sorted by pk" -``next_url`` - Full URL for the next page of results - - ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` - - ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. - - .. code-block:: json - - "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" - ``columns`` List of column names returned by this table, row or query. @@ -402,6 +491,25 @@ The available table extras are listed below. "pk" ] +``column_details`` + SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) + + ``GET /fixtures/binary_data.json?_size=0&_extra=column_details`` + + .. code-block:: json + + { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": false, + "default": null, + "is_pk": false, + "pk_position": 0, + "hidden": 0 + } + } + ``display_columns`` Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys. @@ -807,6 +915,25 @@ The following extras are available for row JSON responses. "id" ] +``column_details`` + SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) + + ``GET /fixtures/binary_data/1.json?_extra=column_details`` + + .. code-block:: json + + { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": false, + "default": null, + "is_pk": false, + "pk_position": 0, + "hidden": 0 + } + } + ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) @@ -1137,7 +1264,6 @@ The following extras are available for arbitrary SQL query responses and stored, "description_html": null, "hide_sql": false, "fragment": null, - "params": [], "parameters": [], "is_write": false, "is_private": false, @@ -1201,6 +1327,23 @@ The following extras are available for arbitrary SQL query responses and stored, .. [[[end]]] +.. _TableCountView: + +Counting all matching rows +-------------------------- + +``POST //
      /-/count`` returns an exact count of the rows matching the table's query string filters:: + + POST /fixtures/facetable/-/count?state=CA + + {"ok": true, "count": 10} + +The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count. + +This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters. + +Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error. + .. _TableAutocompleteView: Table autocomplete @@ -1532,6 +1675,12 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +The request body is always parsed as JSON, regardless of the request's ``Content-Type`` header - a body that is not valid JSON returns a ``400`` error. Cross-site request forgery is prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` header checks rather than by content type requirements. + +The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data. + +Structured inserts, upserts, updates and deletes only support ordinary SQLite tables. Virtual tables and their internal shadow tables are rejected, including when adding rows to an existing table through the create-table API. Writes to ordinary content tables can still update full-text search indexes through configured triggers. + .. _ExecuteWriteView: Executing write SQL @@ -1565,7 +1714,7 @@ Unsupported SQL operations are rejected by default. ``VACUUM`` is not allowed in A successful response includes a message, the SQLite ``rowcount``, a ``"rows"`` list, a ``"truncated"`` flag and a summary of the operations that were executed: -The shape of the ``"analysis"`` block is not yet considered a stable API and may change in future Datasette releases. +The shape of the ``"analysis"`` block is not part of the :ref:`stable API ` and may change in future Datasette releases. .. code-block:: json @@ -1625,15 +1774,17 @@ the execute-write returning row limit, which defaults to 10: ] } -Errors use the standard Datasette error format: +Errors use the :ref:`standard Datasette error format `: .. code-block:: json { "ok": false, + "error": "Permission denied: need execute-write-sql", "errors": [ "Permission denied: need execute-write-sql" - ] + ], + "status": 403 } .. _TableInsertView: @@ -1660,6 +1811,8 @@ A single row can be inserted using the ``"row"`` key: } } +Column values can use the :ref:`binary value JSON format ` to write BLOB data. + If successful, this will return a ``201`` status code and the newly inserted row, for example: .. code-block:: json @@ -1727,9 +1880,11 @@ If any of your rows have a primary key that is already in use, you will get an e { "ok": false, + "error": "UNIQUE constraint failed: new_table.id", "errors": [ "UNIQUE constraint failed: new_table.id" - ] + ], + "status": 400 } Pass ``"ignore": true`` to ignore these errors and insert the other rows: @@ -1765,6 +1920,8 @@ An upsert is an insert or update operation. If a row with a matching primary key The upsert API is mostly the same shape as the :ref:`insert API `. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions. +It also accepts the same :ref:`binary value JSON format `. + :: POST //
      /-/upsert @@ -1802,7 +1959,7 @@ The above example will: Similar to ``/-/insert``, a ``row`` key with an object can be used instead of a ``rows`` array to upsert a single row. -If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. +If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. This is deliberately different from the ``201`` returned by :ref:`insert `: an upsert may update existing rows without creating anything, so it does not claim resource creation. Add ``"return": true`` to the request body to return full copies of the affected rows after they have been inserted or updated: @@ -1859,9 +2016,11 @@ When using upsert you must provide the primary key column (or columns if the tab { "ok": false, + "error": "Row 0 is missing primary key column(s): \"id\"", "errors": [ "Row 0 is missing primary key column(s): \"id\"" - ] + ], + "status": 400 } If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. @@ -1895,6 +2054,8 @@ To update a row, make a ``POST`` to ``//
      //-/update``. You only need to pass the columns you want to update. Any other columns will be left unchanged. +Updated values can use the :ref:`binary value JSON format `. + If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. Add ``"return": true`` to the request body to return the updated row: @@ -1914,14 +2075,16 @@ The returned JSON will look like this: { "ok": true, - "row": { - "id": 1, - "title": "New title", - "other_column": "Will be present here too" - } + "rows": [ + { + "id": 1, + "title": "New title", + "other_column": "Will be present here too" + } + ] } -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. @@ -1942,7 +2105,7 @@ To delete a row, make a ``POST`` to ``//
      //-/delete``. If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableCreateView: @@ -2098,6 +2261,8 @@ Datasette will create a table with a schema that matches those rows and insert t "pk": "id" } +Example rows can use the :ref:`binary value JSON format `, allowing Datasette to infer ``BLOB`` columns. + Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions. The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: @@ -2122,9 +2287,11 @@ If you pass a row to the create endpoint with a primary key that already exists { "ok": false, + "error": "UNIQUE constraint failed: creatures.id", "errors": [ "UNIQUE constraint failed: creatures.id" - ] + ], + "status": 400 } You can avoid this error by passing the same ``"ignore": true`` or ``"replace": true`` options to the create endpoint as you can to the :ref:`insert endpoint `. @@ -2360,7 +2527,7 @@ A successful response returns the new schema and the previous schema. If the req "operations_applied": 11 } -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableSetColumnTypeView: @@ -2424,7 +2591,7 @@ To clear an existing column type assignment, set ``column_type`` to ``null``: This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableDropView: @@ -2461,4 +2628,4 @@ If you pass the following POST body: Then the table will be dropped and a status ``200`` response of ``{"ok": true}`` will be returned. -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 422e67f4..9a4dba23 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -46,7 +46,7 @@ def table_extras(cog): cog.out("\n") for scope, heading, intro, classes in classes_by_scope: cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) - cog.out("{}\n\n".format(intro)) + cog.out(f"{intro}\n\n") for cls in classes: examples = _examples_for_scope(cls, scope) description = cls.description or "" @@ -58,16 +58,16 @@ def table_extras(cog): if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out("``{}``\n".format(cls.key())) - cog.out(" {}\n\n".format(description)) + cog.out(f"``{cls.key()}``\n") + cog.out(f" {description}\n\n") for example in examples: if example.path: value = live_examples[(example.path, example.key or cls.key())] - cog.out(" ``GET {}``\n\n".format(example.path)) + cog.out(f" ``GET {example.path}``\n\n") else: value = example.value if example.note: - cog.out(" {}\n\n".format(example.note)) + cog.out(f" {example.note}\n\n") cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) cog.out("\n\n") @@ -139,7 +139,7 @@ async def _fetch_live_examples(scoped_classes): response = await datasette.client.get(example.path) assert response.status_code == 200, example.path data = response.json() - assert key in data, "{} missing from {}".format(key, example.path) + assert key in data, f"{key} missing from {example.path}" examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 031b3ddd..1bf17f8e 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,7 +1,8 @@ import json import textwrap -from yaml import safe_dump + from ruamel.yaml import YAML +from yaml import safe_dump def metadata_example(cog, data=None, yaml=None): @@ -33,10 +34,10 @@ def config_example( else: data = input output_yaml = safe_dump(input, sort_keys=False) - cog.out("\n.. tab:: {}\n\n".format(yaml_title)) + cog.out(f"\n.. tab:: {yaml_title}\n\n") cog.out(" .. code-block:: yaml\n\n") cog.out(textwrap.indent(output_yaml, " ")) - cog.out("\n\n.. tab:: {}\n\n".format(json_title)) + cog.out(f"\n\n.. tab:: {json_title}\n\n") cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") @@ -44,9 +45,11 @@ def config_example( def internal_schema(cog): import asyncio - from datasette.app import Datasette + from sqlite_utils import Database + from datasette.app import Datasette + ds = Datasette() db = ds.get_internal_database() diff --git a/docs/pages.rst b/docs/pages.rst index ce88be12..65c03a49 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -95,7 +95,7 @@ Use the :ref:`ExecuteWriteView` JSON API to execute writable SQL programmaticall Stored query browsers --------------------- -The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. +The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. The JSON versions accept ``?_size=`` (default 50, ``max`` for the :ref:`setting_max_returned_rows` limit) and a ``?_next=`` pagination token. These pages support search, pagination and filters for read-only or writable queries and private or public queries. Adding a ``.json`` extension to either URL returns the same list as JSON. @@ -169,11 +169,13 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi .. code-block:: json { + "ok": true, "schemas": [ { "database": "content", "schema": "create table posts ..." } + ] } .. _DatabaseSchemaView: @@ -181,11 +183,11 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi Database schema --------------- -Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"`` and ``"schema"`` keys. +Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"`` and ``"schema"`` keys. .. _TableSchemaView: Table schema ------------ -Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"``, ``"table"``, and ``"schema"`` keys. +Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"``, ``"table"``, and ``"schema"`` keys. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 81ef4acd..3efff7a4 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -217,7 +217,7 @@ Extra template variables that should be made available in the rendered template ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` -This hook can return one of three different types: +This hook supports the following return values: Dictionary If you return a dictionary its keys and values will be merged into the template context. @@ -228,6 +228,9 @@ Function that returns a dictionary Function that returns an awaitable function that returns a dictionary You can also return a function which returns an awaitable function which returns a dictionary. +``None`` + The hook itself, or a function or awaitable it returns, can return ``None`` when no extra variables are needed. Variables returned by other plugins are still included. + Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template. .. warning:: @@ -254,8 +257,6 @@ This example returns an awaitable function which adds a list of ``hidden_table_n return { "hidden_table_names": await db.hidden_table_names() } - else: - return {} return hidden_table_names @@ -495,10 +496,10 @@ Lets you customize the display of values within table cells in the HTML table vi The name of the column being rendered ``table`` - string or None - The name of the table - or ``None`` if this is a custom SQL query + The name of the table or view - or ``None`` if this is a custom SQL query ``pks`` - list of strings - The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views (where ``table`` is ``None``), this will be an empty list ``[]``. + The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views, this will be an empty list ``[]``. ``database`` - string The name of the database @@ -1107,7 +1108,7 @@ Return an `ASGI `__ middleware wrapper function th This is a very powerful hook. You can use it to manipulate the entire Datasette response, or even to configure new URL routes that will be handled by your own custom code. -You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette `__. +You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette `__. This example plugin adds a ``x-databases`` HTTP header listing the currently attached databases: @@ -1157,7 +1158,7 @@ Examples: `datasette-cors `__, `dat startup(datasette) ------------------ -This hook fires when the Datasette application server first starts up. +This hook fires when the Datasette application server first starts up. It runs on the same event loop that goes on to serve requests, so it is safe to create loop-bound primitives and register background work here — see :ref:`datasette_lifecycle` for the full guarantee and the three ways startup can be triggered. Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails: @@ -1195,6 +1196,7 @@ Potential use-cases: * Create database tables that a plugin needs on startup * Validate the configuration for a plugin on startup, and raise an error if it is invalid * Raise a ``datasette.utils.StartupError("message")`` exception to prevent Datasette from starting and display that message to the user. +* Register supervised long-lived background work using :ref:`datasette_add_background_task`, which core launches once every plugin's ``startup()`` hook has finished. .. note:: @@ -1211,6 +1213,31 @@ Potential use-cases: Examples: `datasette-saved-queries `__, `datasette-init `__ +.. _plugin_hook_shutdown: + +shutdown(datasette) +------------------- + +This hook fires once, when the Datasette application server is shutting down gracefully - triggered by the ASGI ``lifespan.shutdown`` event, which includes pressing Ctrl-C or sending ``SIGTERM`` to a ``datasette serve`` process. It is not called on a hard kill (``SIGKILL``), since there is no opportunity to run any code in that case. + +Like ``startup()``, this can be a regular function or it can return an async function to be awaited. + +It runs before Datasette cancels any background tasks it is supervising (see :ref:`datasette_add_background_task`) and before it closes its database connections, so you can use it to tell your plugin's own background work to stop gracefully while a database connection is still available to write out any final state. See :ref:`datasette_lifecycle` for exactly where this fits into the full startup-to-shutdown sequence: + +.. code-block:: python + + @hookimpl + def shutdown(datasette): + async def inner(): + db = datasette.get_database() + await db.execute_write( + "insert into shutdown_log (at) values (datetime('now'))" + ) + + return inner + +If your ``shutdown()`` hook raises an exception it will be logged but not re-raised, so one plugin's broken shutdown code cannot prevent other plugins - or Datasette itself - from finishing their own teardown. + .. _plugin_hook_actor_from_request: actor_from_request(datasette, request) @@ -1685,6 +1712,8 @@ forbidden(datasette, request, message) Plugins can use this to customize how Datasette responds when a 403 Forbidden error occurs - usually because a page failed a permission check, see :ref:`authentication_permissions`. +Datasette's default behavior returns the :ref:`standard JSON error format ` with a 403 status when the request path ends in ``.json`` or the request has an ``Accept: application/json`` or ``Content-Type: application/json`` header; other requests get an HTML error page. + If a plugin hook wishes to react to the error, it should return a :ref:`Response object `. This example returns a redirect to a ``/-/login`` page: @@ -2544,6 +2573,10 @@ The default ``SignedTokenHandler`` uses itsdangerous signed tokens (``dstok_`` p async def verify_token(self, datasette, token): # Look up token in database, return actor dict or None + # if this handler does not recognize the token. Raise + # datasette.TokenInvalid for a token this handler + # recognizes but rejects (revoked, expired) - Datasette + # will respond with a 401 error. ... diff --git a/docs/plugin_telemetry.rst b/docs/plugin_telemetry.rst new file mode 100644 index 00000000..e4c7381d --- /dev/null +++ b/docs/plugin_telemetry.rst @@ -0,0 +1,270 @@ +.. _plugin_telemetry: + +Telemetry for plugin authors +============================ + +Datasette core emits OpenTelemetry spans and metrics for the work it does itself - see :ref:`internals_telemetry` for what those are and how an operator turns them on. This page is about the other half: instrumenting the work **your plugin** does, so that a plugin's queries, background jobs and custom operations show up in the same traces and the same metrics pipeline, using the same conventions. + +.. _plugin_telemetry_scope: + +Use your own instrumentation scope +---------------------------------- + +Create a tracer and meter using your plugin's own instrumentation scope: + +.. code-block:: python + + from opentelemetry import metrics, trace + + from my_plugin import __version__ + + tracer = trace.get_tracer("my_plugin", __version__) + meter = metrics.get_meter("my_plugin", __version__) + +Use these naming rules: + +- **Scope**: use your plugin's import package name, such as ``my_plugin``. This lets users filter telemetry by plugin. +- **Signal prefix**: prefix spans, metrics and custom attributes with your package name (``my_plugin.*``) or a product name (``paper.*``). The ``datasette.*`` prefix is reserved for core. + +Reuse shared attribute names where they describe the same thing: ``db.namespace`` for a database name, or ``error.type`` for an exception class. + +If you pass ``schema_url=`` when creating a tracer or meter, choose the semantic-convention version that matches your attributes. Datasette's version is available as ``datasette.telemetry.SCHEMA_URL``. Omit ``schema_url`` if you are unsure which version applies. + +.. _plugin_telemetry_registry: + +Declare a registry +------------------ + +Use ``Attribute``, ``SpanName`` and ``MetricName`` from ``datasette.telemetry_registry`` to describe your plugin's telemetry. Registry entries are strings and can be passed directly to OpenTelemetry: + +.. code-block:: python + + from datasette.telemetry_registry import ( + Attribute, + MetricName, + SpanName, + ) + + OUTCOME = Attribute( + "my_plugin.outcome", + "How the job ended.", + values={"ok", "error", "skipped"}, + ) + JOB_NAME = Attribute( + "my_plugin.job", "The registered job name." + ) + + JOB_RUN = SpanName( + "my_plugin.job.run", + "One execution of a scheduled job.", + (OUTCOME, JOB_NAME), + ) + + # A span family with a variable suffix - emitted as "my_plugin.chat gpt-5" + CHAT = SpanName( + "my_plugin.chat ", + "One model call, named ``my_plugin.chat {model}``.", + prefix=True, + ) + + SPANS = (JOB_RUN, CHAT) + + JOB_DURATION = MetricName( + "my_plugin.job.duration", + "Histogram", + "s", + "How long each job took.", + (JOB_NAME, OUTCOME), + buckets=(0.01, 0.1, 1, 10, 60, 600, 3600), + ) + + METRICS = (JOB_DURATION,) + +The example uses these optional arguments: + +``values`` - iterable + Allowed values for an ``Attribute``. The :ref:`conformance helpers ` check that emitted values belong to this set. Omit it to allow any value. + +``prefix`` - boolean + For ``SpanName``, match emitted names by prefix. Defaults to ``False``. Exact names take precedence over prefix matches. Avoid overlapping prefixes: the first matching entry in the registry wins. + +``buckets`` - iterable + Histogram boundaries for a ``MetricName``, expressed in the metric's unit. Pass these to ``meter.create_histogram()`` using ``explicit_bucket_boundaries_advisory=JOB_DURATION.buckets``. Choose boundaries suitable for the operations you measure. For SQLite timings, ``datasette.telemetry_registry.DURATION_BUCKETS`` provides boundaries from 0.0001 to 10 seconds. + +.. _plugin_telemetry_privacy: + +Privacy and cardinality rules +----------------------------- + +Core does not explicitly attach bound SQL parameter values, actor identifiers, cookies, authorization headers, client IP addresses or URL query strings as attributes. It does record SQL text, URL paths, host names, User-Agent headers and exception details, which may contain sensitive information. See :ref:`internals_telemetry_privacy`. + +- Prefer closed enums, booleans, counts and durations for attribute values. Avoid recording personal information, tokens or other secrets. +- If you record SQL, use ``datasette.telemetry.sql_attribute()`` on spans only. It truncates SQL text but does not redact literal values. Do not add bound parameter values. +- Keep metric dimensions bounded. For user input or other unbounded values, record a count, a byte size, a truncation flag or an enum outcome instead. + +Use ``assert_no_forbidden_values()`` in :ref:`plugin_telemetry_testing` to check for specific sensitive values in captured telemetry. This helper does not automatically identify all sensitive information. + +.. _plugin_telemetry_callbacks: + +Your database work is already traced +------------------------------------ + +Every call your plugin makes through :ref:`db.execute() `, :ref:`db.execute_fn() `, :ref:`db.execute_write() ` and :ref:`db.execute_write_fn() ` already emits core's ``db.query`` spans and is counted in the ``db.client.operation.duration`` histogram. Two consequences: + +- **Pass named callables**, not lambdas: the span for a callback-style call is identified by ``datasette.callback``, the callable's qualified name, and a lambda reports ````. +- If you also wrap those calls in your own span or histogram, you are creating a *second* series in *your* scope - that is fine and sometimes right (yours can carry plugin-level attributes core cannot know), but it is a deliberate two-series design, not a substitute for core's. + +.. _plugin_telemetry_request_span: + +Enriching the request span +-------------------------- + +Inside a view or ASGI middleware, ``datasette.telemetry.request_span(scope)`` returns the recording ``SERVER`` span for the current request, or ``None`` when nothing is recording - which is also your signal to skip any work done only to compute attributes: + +.. code-block:: python + + from datasette.telemetry import request_span + + + async def my_view(request): + span = request_span(request.scope) + if span is not None: + span.set_attribute("my_plugin.cache", "hit") + ... + +.. _plugin_telemetry_background: + +Background work: roots with links +--------------------------------- + +For background work that can outlive a request, create a root span linked to the span that scheduled it. Call ``linked_root_span_kwargs()`` when scheduling the work, then pass the result when starting its span. If there is no valid span context to capture, the new span has no link: + +.. code-block:: python + + from datasette.telemetry import linked_root_span_kwargs + + # Capture the current span when scheduling the work: + kwargs = linked_root_span_kwargs() + + # Later, wherever the work actually runs: + with tracer.start_as_current_span( + "my_plugin.job.run", **kwargs + ) as span: + span.set_attribute(OUTCOME, "ok") + +For periodic tasks, create a root span and increment a counter on each iteration, including iterations with no work. Record the result in an outcome attribute. A gauge reporting the time since the last iteration can help monitor tasks with long intervals. + +``asyncio.create_task()`` inherits the current trace context. Use ``linked_root_span_kwargs()`` to start background work with its own root span and a link to that context. + +Tracers and meters can be created at module scope. In embedded deployments, configure the application's providers before the work you want to record begins. + +.. _plugin_telemetry_gauges: + +Observable gauges +----------------- + +Use an observable gauge for current values such as the number of open streams or the length of a queue. The SDK calls its callback when collecting metrics: + +- Track live objects using weak references, such as a ``weakref.WeakSet``, and unregister them when they close. +- Callbacks may run on a different thread from request handlers. Protect shared state and avoid waiting on locks held by request handlers. +- Read cached state and yield ``Observation`` values. Keep callbacks synchronous and free of I/O. Refresh cached values outside the callback; use a separate gauge to report their age if needed. + +Without a provider, gauge callbacks are not invoked. + +.. _plugin_telemetry_testing: + +Testing your instrumentation +---------------------------- + +Use ``datasette.telemetry_testing`` to capture telemetry in your tests and check it against your registry. Add `opentelemetry-sdk `__ to your test dependencies, then import these fixtures in ``conftest.py``: + +.. code-block:: python + + from datasette.telemetry_testing import ( # noqa: F401 + otel_metrics, + otel_meter_provider, + otel_provider, + otel_reset, + otel_spans, + ) + +``otel_provider`` and ``otel_meter_provider`` + Automatically configure in-memory recording for spans and metrics once per test session. + +``otel_reset`` + Automatically clears recorded spans and drains collected metrics after every test. + +``otel_spans`` + Provides an ``InMemorySpanExporter``. Call ``get_finished_spans()`` to retrieve spans recorded during the test. + +``otel_metrics`` + Provides a metrics collector. Call ``collect()`` to capture a snapshot, then use ``point()`` or ``points()`` to inspect it. + +Tests requesting ``otel_spans`` or ``otel_metrics`` skip if the SDK is unavailable or another provider has already been installed. + +The assertion helpers check the recorded telemetry against your registry: + +``assert_spans_conform()`` + Checks that emitted spans and attributes are registered, and attribute values match any declared ``values=`` enums. + +``assert_metrics_conform()`` + Checks that emitted metrics and attributes are registered, attribute values match any declared enums, and instrument kinds and units match the registry. + +``assert_spans_covered()`` and ``assert_metrics_covered()`` + Check that every registered span or metric and its required attributes appeared during the test. Attributes marked ``optional=True`` are excluded from this check; test those separately. + +Pass your plugin's instrumentation scope as ``scope_name`` to these helpers, since the fixtures also record Datasette's own telemetry. + +Run a workload that exercises your instrumentation, then call ``otel_metrics.collect()`` once before checking the metrics. Counters and histograms report measurements since the previous collection. Keep the Datasette instance open until collection so observable gauges can report its state: + +.. code-block:: python + + from datasette.telemetry_testing import ( + assert_metrics_conform, + assert_metrics_covered, + assert_package_never_imports_sdk, + assert_spans_covered, + assert_spans_conform, + ) + + from my_plugin.telemetry import METRICS, SPANS + + + def test_api_only_dependency(): + assert_package_never_imports_sdk("my_plugin") + + + def test_conformance(otel_spans, otel_metrics): + run_a_workload_that_exercises_everything() + finished = otel_spans.get_finished_spans() + # Everything emitted is registered (and enum values are legal): + assert_spans_conform( + SPANS, finished, scope_name="my_plugin" + ) + # Everything registered was emitted: + assert_spans_covered( + SPANS, finished, scope_name="my_plugin" + ) + # Collect once, then check the metrics: + otel_metrics.collect() + assert_metrics_conform( + METRICS, otel_metrics, scope_name="my_plugin" + ) + assert_metrics_covered( + METRICS, otel_metrics, scope_name="my_plugin" + ) + +``assert_package_never_imports_sdk()`` checks that importing your plugin does not import the OpenTelemetry SDK. Run this test early in your suite; see the helper's docstring for a macOS threading limitation. + +Use ``assert_no_forbidden_values()`` to check for private data in telemetry. Include fake email addresses, tokens or usernames in your test workload, then pass those values, the finished spans and the collected metrics to the helper. It checks span names, attributes, events, status descriptions and metric attributes. + +Leave ``scope_name`` unset for privacy checks so they include both your plugin's telemetry and Datasette's own. + +.. _plugin_telemetry_caveats: + +Known caveats +------------- + +- **Streaming responses hold the request span open.** Core's request span ends when the response body finishes, so for an SSE or long-streaming route its duration is the connection lifetime. If you need per-message timing on a stream, emit your own child spans or span events per message, and use gauges for concurrent-stream counts. +- **A plugin timing core's work double-measures by design.** See :ref:`plugin_telemetry_callbacks` above. +- ``datasette.client`` requests made from inside a request produce a nested ``SERVER`` span. Those spans carry ``datasette.internal_client: true`` - filter on it to keep kind-based dashboards from double-counting requests. diff --git a/docs/plugins.rst b/docs/plugins.rst index d2b5c20a..296ef55d 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -261,6 +261,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "permission_resources_sql" ] }, + { + "name": "datasette.default_permissions.sqlite_statistics", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "permission_resources_sql" + ] + }, { "name": "datasette.default_permissions.tokens", "static": false, @@ -459,6 +468,8 @@ Secret configuration values Some plugins may need configuration that should stay secret - API keys for example. There are two ways in which you can store secret configuration values. +The :ref:`/-/config ` introspection endpoint redacts the values of any configuration keys whose names contain one of these substrings: ``secret``, ``key``, ``password``, ``token``, ``hash`` or ``dsn``. Name your plugin's secret configuration keys accordingly - for example ``api_key`` or ``client_secret`` - so they are automatically redacted there. + **As environment variables**. If your secret lives in an environment variable that is available to the Datasette process, you can indicate that the configuration value should be read from that environment variable like so: .. [[[cog diff --git a/docs/settings.rst b/docs/settings.rst index 5cd49113..f3e6636d 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -67,10 +67,21 @@ The following options can be set using ``--setting name value``, or by storing t default_allow_sql ~~~~~~~~~~~~~~~~~ +.. [[[cog + from settings_doc import setting_default + setting_default(cog, "default_allow_sql") +.. ]]] + +Default: ``on`` + +.. [[[end]]] + Should users be able to execute arbitrary SQL queries by default? Setting this to ``off`` causes permission checks for :ref:`actions_execute_sql` to fail by default. +This setting controls the ability to submit arbitrary SQL. It does not disable structured table-browsing features that use SQL generated by Datasette, such as sorting, column filters and :ref:`facets`. Use :ref:`setting_allow_facet` to control whether users can request facets. + :: datasette mydatabase.db --setting default_allow_sql off @@ -82,6 +93,14 @@ Another way to achieve this is to add ``"allow_sql": false`` to your ``datasette default_page_size ~~~~~~~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "default_page_size") +.. ]]] + +Default: ``100`` + +.. [[[end]]] + The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` query string parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--setting`` like so:: datasette mydatabase.db --setting default_page_size 50 @@ -91,7 +110,15 @@ The default number of rows returned by the table page. You can over-ride this on sql_time_limit_ms ~~~~~~~~~~~~~~~~~ -By default, queries have a time limit of one second. If a query takes longer than this to run Datasette will terminate the query and return an error. +.. [[[cog + setting_default(cog, "sql_time_limit_ms") +.. ]]] + +Default: ``1000`` + +.. [[[end]]] + +Time limit for SQL queries, in milliseconds. If a query takes longer than this to run Datasette will terminate the query and return an error. If this time limit is too short for you, you can customize it using the ``sql_time_limit_ms`` limit - for example, to increase it to 3.5 seconds:: @@ -108,7 +135,15 @@ This would set the time limit to 100ms for that specific query. This feature is max_returned_rows ~~~~~~~~~~~~~~~~~ -Datasette returns a maximum of 1,000 rows of data at a time. If you execute a query that returns more than 1,000 rows, Datasette will return the first 1,000 and include a warning that the result set has been truncated. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more than 1,000 rows. +.. [[[cog + setting_default(cog, "max_returned_rows") +.. ]]] + +Default: ``1000`` + +.. [[[end]]] + +The maximum number of rows Datasette returns at a time. If you execute a query that exceeds this limit, Datasette will truncate the result set and include a warning. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more rows. You can increase or decrease this limit like so:: @@ -119,18 +154,59 @@ You can increase or decrease this limit like so:: max_insert_rows ~~~~~~~~~~~~~~~ -Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100. +.. [[[cog + setting_default(cog, "max_insert_rows") +.. ]]] + +Default: ``100`` + +.. [[[end]]] + +Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_insert_rows 1000 +.. _setting_max_post_body_bytes: + +max_post_body_bytes +~~~~~~~~~~~~~~~~~~~ + +.. [[[cog + setting_default(cog, "max_post_body_bytes") +.. ]]] + +Default: ``2097152`` + +.. [[[end]]] + +Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API `. Requests with larger bodies are rejected with an HTTP 413 error. + +This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over. + +If you increase :ref:`setting_max_insert_rows` to support larger bulk inserts you may need to increase this limit as well:: + + datasette mydatabase.db --setting max_post_body_bytes 10485760 + +Set it to 0 to disable the limit entirely:: + + datasette mydatabase.db --setting max_post_body_bytes 0 + .. _setting_num_sql_threads: num_sql_threads ~~~~~~~~~~~~~~~ -Maximum number of threads in the thread pool Datasette uses to execute SQLite queries. Defaults to 3. +.. [[[cog + setting_default(cog, "num_sql_threads") +.. ]]] + +Default: ``3`` + +.. [[[end]]] + +Maximum number of threads in the thread pool Datasette uses to execute SQLite queries. :: @@ -143,9 +219,17 @@ Setting this to 0 turns off threaded SQL queries entirely - useful for environme allow_facet ~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "allow_facet") +.. ]]] + +Default: ``on`` + +.. [[[end]]] + Allow users to specify columns they would like to facet on using the ``?_facet=COLNAME`` URL parameter to the table view. -This is enabled by default. If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table. +If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table. Here's how to disable this feature:: @@ -156,7 +240,15 @@ Here's how to disable this feature:: default_facet_size ~~~~~~~~~~~~~~~~~~ -The default number of unique rows returned by :ref:`facets` is 30. You can customize it like this:: +.. [[[cog + setting_default(cog, "default_facet_size") +.. ]]] + +Default: ``30`` + +.. [[[end]]] + +The default number of unique rows returned by :ref:`facets`. You can customize it like this:: datasette mydatabase.db --setting default_facet_size 50 @@ -165,7 +257,15 @@ The default number of unique rows returned by :ref:`facets` is 30. You can custo facet_time_limit_ms ~~~~~~~~~~~~~~~~~~~ -This is the time limit Datasette allows for calculating a facet, which defaults to 200ms:: +.. [[[cog + setting_default(cog, "facet_time_limit_ms") +.. ]]] + +Default: ``200`` + +.. [[[end]]] + +The time limit in milliseconds Datasette allows for calculating a facet. You can customize it like this:: datasette mydatabase.db --setting facet_time_limit_ms 1000 @@ -174,7 +274,15 @@ This is the time limit Datasette allows for calculating a facet, which defaults facet_suggest_time_limit_ms ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. The default for this time limit is 50ms to account for the fact that it needs to run once for every column. If the time limit is exceeded the column will not be suggested as a facet. +.. [[[cog + setting_default(cog, "facet_suggest_time_limit_ms") +.. ]]] + +Default: ``50`` + +.. [[[end]]] + +When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. This time limit, in milliseconds, applies separately to each query. If the time limit is exceeded the column will not be suggested as a facet. You can increase this time limit like so:: @@ -185,7 +293,15 @@ You can increase this time limit like so:: suggest_facets ~~~~~~~~~~~~~~ -Should Datasette calculate suggested facets? On by default, turn this off like so:: +.. [[[cog + setting_default(cog, "suggest_facets") +.. ]]] + +Default: ``on`` + +.. [[[end]]] + +Should Datasette calculate suggested facets? Turn this off like so:: datasette mydatabase.db --setting suggest_facets off @@ -194,7 +310,15 @@ Should Datasette calculate suggested facets? On by default, turn this off like s allow_download ~~~~~~~~~~~~~~ -Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default. However, databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following:: +.. [[[cog + setting_default(cog, "allow_download") +.. ]]] + +Default: ``on`` + +.. [[[end]]] + +Should users be able to download the original SQLite database using a link on the database index page? Databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following:: datasette mydatabase.db --setting allow_download off @@ -203,9 +327,17 @@ Should users be able to download the original SQLite database using a link on th allow_signed_tokens ~~~~~~~~~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "allow_signed_tokens") +.. ]]] + +Default: ``on`` + +.. [[[end]]] + Should users be able to create signed API tokens to access Datasette? -This is turned on by default. Use the following to turn it off:: +Use the following to turn it off:: datasette mydatabase.db --setting allow_signed_tokens off @@ -216,9 +348,17 @@ Turning this setting off will disable the ``/-/create-token`` page, :ref:`descri max_signed_tokens_ttl ~~~~~~~~~~~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "max_signed_tokens_ttl") +.. ]]] + +Default: ``0`` + +.. [[[end]]] + Maximum allowed expiry time for signed API tokens created by users. -Defaults to ``0`` which means no limit - tokens can be created that will never expire. +A value of ``0`` means no limit - tokens can be created that will never expire. Set this to a value in seconds to limit the maximum expiry time. For example, to set that limit to 24 hours you would use:: @@ -231,18 +371,36 @@ This setting is enforced when incoming tokens are processed. default_cache_ttl ~~~~~~~~~~~~~~~~~ -Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely. Defaults to 5 seconds. +.. [[[cog + setting_default(cog, "default_cache_ttl") +.. ]]] + +Default: ``5`` + +.. [[[end]]] + +Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely. :: datasette mydatabase.db --setting default_cache_ttl 60 +Dynamic responses for authenticated actors, requests with cookies or an ``Authorization`` header, and responses that set cookies use ``Cache-Control: private, no-store``. This takes precedence over ``default_cache_ttl`` and ``?_ttl=``, even when cache headers are otherwise disabled. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``. Static assets retain their own cache policy. + .. _setting_cache_size_kb: cache_size_kb ~~~~~~~~~~~~~ -Sets the amount of memory SQLite uses for its `per-connection cache `_, in KB. +.. [[[cog + setting_default(cog, "cache_size_kb") +.. ]]] + +Default: ``0`` + +.. [[[end]]] + +Sets the amount of memory SQLite uses for its `per-connection cache `_, in KB. Set this to ``0`` to use SQLite's default cache size. :: @@ -253,9 +411,17 @@ Sets the amount of memory SQLite uses for its `per-connection cache ` where an entire table (potentially hundreds of thousands of rows) can be exported as a single CSV -file. This is turned on by default - you can turn it off like this: +file. You can turn it off like this: :: @@ -266,8 +432,16 @@ file. This is turned on by default - you can turn it off like this: max_csv_mb ~~~~~~~~~~ -The maximum size of CSV that can be exported, in megabytes. Defaults to 100MB. -You can disable the limit entirely by settings this to 0: +.. [[[cog + setting_default(cog, "max_csv_mb") +.. ]]] + +Default: ``100`` + +.. [[[end]]] + +The maximum size of CSV that can be exported, in megabytes. +You can disable the limit entirely by setting this to 0: :: @@ -278,6 +452,14 @@ You can disable the limit entirely by settings this to 0: truncate_cells_html ~~~~~~~~~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "truncate_cells_html") +.. ]]] + +Default: ``2048`` + +.. [[[end]]] + In the HTML table view, truncate any strings that are longer than this value. The full value will still be available in CSV, JSON and on the individual row HTML page. Set this to 0 to disable truncation. @@ -291,6 +473,14 @@ HTML page. Set this to 0 to disable truncation. force_https_urls ~~~~~~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "force_https_urls") +.. ]]] + +Default: ``off`` + +.. [[[end]]] + Forces self-referential URLs in the JSON output to always use the ``https://`` protocol. This is useful for cases where the application itself is hosted using HTTP but is served to the outside world via a proxy that enables HTTPS. @@ -304,6 +494,14 @@ HTTP but is served to the outside world via a proxy that enables HTTPS. template_debug ~~~~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "template_debug") +.. ]]] + +Default: ``off`` + +.. [[[end]]] + This setting enables template context debug mode, which is useful to help understand what variables are available to custom templates when you are writing them. Enable it like this:: @@ -323,6 +521,14 @@ Some examples: trace_debug ~~~~~~~~~~~ +.. [[[cog + setting_default(cog, "trace_debug") +.. ]]] + +Default: ``off`` + +.. [[[end]]] + This setting enables appending ``?_trace=1`` to any page in order to see the SQL queries and other trace information that was used to generate that page. Enable it like this:: @@ -341,6 +547,14 @@ See :ref:`internals_tracer` for details on how to hook into this mechanism as a base_url ~~~~~~~~ +.. [[[cog + setting_default(cog, "base_url") +.. ]]] + +Default: ``/`` + +.. [[[end]]] + If you are running Datasette behind a proxy, it may be useful to change the root path used for the Datasette instance. For example, if you are sending traffic from ``https://www.example.com/tools/datasette/`` through to a proxied Datasette instance you may wish Datasette to use ``/tools/datasette/`` as its root URL. diff --git a/docs/settings_doc.py b/docs/settings_doc.py new file mode 100644 index 00000000..e041f333 --- /dev/null +++ b/docs/settings_doc.py @@ -0,0 +1,10 @@ +"""Cog helper for documenting setting defaults from Datasette's registry.""" + + +def setting_default(cog, name): + from datasette.app import DEFAULT_SETTINGS + + default = DEFAULT_SETTINGS[name] + if isinstance(default, bool): + default = "on" if default else "off" + cog.out(f"\nDefault: ``{default}``\n\n") diff --git a/docs/shots.yml b/docs/shots.yml new file mode 100644 index 00000000..0a5f9fa6 --- /dev/null +++ b/docs/shots.yml @@ -0,0 +1,38 @@ +# Screenshots used by the documentation, taken using shot-scraper: +# https://shot-scraper.datasette.io/en/stable/multi.html +# +# Run "just shots" from the repository root to create any that are +# missing. Existing images are skipped, so delete an image to recreate it. +# +# Paths are relative to this docs/ directory. + +# Serves the JavaScript in docs/shots/ and loads it on every page. +# List form means the datasette process is stopped directly when done. +- server: + - datasette + - --memory + - --port + - 8755 + - --static + - shots:shots + - -s + - extra_js_urls + - '["/shots/modal-example.js", "/shots/modal-classes.js"]' + +# javascript_plugins.rst - Reusable modal dialogs +- output: images/modal-example.webp + url: http://localhost:8755/ + javascript: | + document.querySelector('[aria-controls="my-plugin-dialog"]').click(); + selector: "#my-plugin-dialog" + padding: 32 + quality: 70 + +- output: images/modal-classes.webp + url: http://localhost:8755/ + javascript: | + document.querySelector('[aria-controls="export-dialog"]').click(); + document.activeElement.blur(); + selector: "#export-dialog" + padding: 32 + quality: 70 diff --git a/docs/shots/modal-classes.js b/docs/shots/modal-classes.js new file mode 100644 index 00000000..57372613 --- /dev/null +++ b/docs/shots/modal-classes.js @@ -0,0 +1,41 @@ +// Demonstrates every shared modal CSS class, for images/modal-classes.webp +document.addEventListener("datasette_init", () => { + const openButton = document.createElement("button"); + openButton.type = "button"; + openButton.textContent = "Open export dialog"; + openButton.setAttribute("aria-haspopup", "dialog"); + openButton.setAttribute("aria-controls", "export-dialog"); + + const modal = DatasetteModal.create(); + const dialog = modal.dialog; + dialog.id = "export-dialog"; + dialog.setAttribute("aria-labelledby", "export-dialog-title"); + dialog.innerHTML = ` +
      + + 3 selected +
      +
      +

      Export these rows from the plants table as CSV:

      +
        +
      • Monstera deliciosa
        • +
      • Ficus lyrata
        • +
      • Pilea peperomioides
        • +
      +
      +
      + CSV, UTF-8 + + +
      `; + + const [cancelButton, exportButton] = dialog.querySelectorAll(".modal-footer button"); + cancelButton.addEventListener("click", () => modal.requestClose("cancel")); + exportButton.addEventListener("click", () => modal.close()); + openButton.addEventListener("click", () => { + modal.show({ returnFocusTo: openButton, initialFocus: exportButton }); + }); + + document.body.append(modal); + document.querySelector("section.content").append(openButton); +}); diff --git a/docs/shots/modal-example.js b/docs/shots/modal-example.js new file mode 100644 index 00000000..fd8210ab --- /dev/null +++ b/docs/shots/modal-example.js @@ -0,0 +1,38 @@ +document.addEventListener("datasette_init", () => { + const openButton = document.createElement("button"); + openButton.type = "button"; + openButton.textContent = "Open example dialog"; + // Indicate that this button opens a dialog: + openButton.setAttribute("aria-haspopup", "dialog"); + // Identify which dialog it controls: + openButton.setAttribute("aria-controls", "my-plugin-dialog"); + + const modal = DatasetteModal.create(); + const dialog = modal.dialog; + dialog.id = "my-plugin-dialog"; + // Tell screenreaders the dialog is labelled by #my-plugin-dialog-title + dialog.setAttribute("aria-labelledby", "my-plugin-dialog-title"); + dialog.innerHTML = ` +
      + +
      +
      + This dialog uses Datasette's shared styles and keyboard behavior. +
      +
      + +
      `; + + const closeButton = dialog.querySelector("button"); + closeButton.addEventListener("click", () => { + modal.requestClose("cancel"); + }); + openButton.addEventListener("click", () => { + modal.show({ returnFocusTo: openButton, initialFocus: closeButton }); + }); + + document.body.append(modal); + document.querySelector("section.content").append(openButton); +}); diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 371348fb..4c6e4426 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -657,7 +657,7 @@ There are three options for specifying that you would like the response to your - Include ``?_json=1`` in the URL that you POST to - Include ``"_json": 1`` in your JSON body, or ``&_json=1`` in your form encoded body -The JSON response will look like this: +A successful JSON response will look like this: .. code-block:: json @@ -667,7 +667,21 @@ The JSON response will look like this: "redirect": "/data/add_name" } -The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. +If the SQL fails to execute - for example a constraint violation - the response uses the :ref:`standard error format ` with a ``400`` status, plus the ``"redirect"`` key from the query configuration: + +.. code-block:: json + + { + "ok": false, + "error": "UNIQUE constraint failed: docs.id", + "errors": [ + "UNIQUE constraint failed: docs.id" + ], + "status": 400, + "redirect": null + } + +The ``"message"``, ``"error"`` and ``"redirect"`` values here take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. .. _pagination: diff --git a/docs/telemetry_doc.py b/docs/telemetry_doc.py new file mode 100644 index 00000000..9c4086ae --- /dev/null +++ b/docs/telemetry_doc.py @@ -0,0 +1,47 @@ +""" +Cog helpers that render the span and metric reference in ``internals.rst`` +from ``datasette/telemetry_registry.py``. +""" + + +def _attribute_lines(cog, attributes): + if not attributes: + cog.out(" No attributes.\n\n") + return + cog.out(" Attributes:\n\n") + for attribute in attributes: + suffix = " *(optional)*" if attribute.optional else "" + line = f" - ``{attribute}``{suffix} - {attribute.description}" + if attribute.values is not None: + rendered = ", ".join(f"``{value}``" for value in sorted(attribute.values)) + line += f" One of: {rendered}." + cog.out(line + "\n") + cog.out("\n") + + +def spans(cog): + from opentelemetry.trace import SpanKind + + from datasette.telemetry_registry import SPANS + + cog.out("\n") + for span in SPANS: + cog.out(f"``{span}``\n") + cog.out(f" {span.description}\n\n") + # Only show the kind for spans that are not INTERNAL + if span.kind != SpanKind.INTERNAL: + cog.out(f" Kind: ``{span.kind.name}``.\n\n") + _attribute_lines(cog, span.attributes) + + +def metrics(cog): + from datasette.telemetry_registry import METRICS + + cog.out("\n") + for metric in METRICS: + cog.out(f"``{metric}``\n") + cog.out(f" {metric.kind}, unit ``{metric.unit}``. {metric.description}\n\n") + if metric.buckets: + boundaries = ", ".join(f"``{boundary}``" for boundary in metric.buckets) + cog.out(f" Bucket boundaries: {boundaries}.\n\n") + _attribute_lines(cog, metric.attributes) diff --git a/docs/template_context.rst b/docs/template_context.rst index 5c6b1567..e445b335 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -98,7 +98,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re The color assigned to the database ``database_page_data`` - ``dict`` - JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``. + JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``. ``editable`` - ``bool`` Boolean indicating if the database is editable @@ -329,7 +329,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. Pagination token for the next page, or None ``next_url`` - ``str`` - Full URL for the next page of results + Full URL for the next page of results, or None if there are no more pages. See :ref:`json_api_pagination`. ``ok`` - ``bool`` True if the data for this page was retrieved without errors @@ -389,7 +389,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. SQL definition for this table ``table_insert_ui`` - ``dict`` - Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys. + Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys. ``table_page_data`` - ``dict`` JSON data used by JavaScript on the table page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs, optional ``insertRow`` data and optional ``alterTable`` data. diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index a5f4fb6f..95b8a366 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -21,14 +21,12 @@ def template_context(cog): ), ) for name, doc in TEMPLATE_BASE_CONTEXT.items(): - cog.out("``{}``\n".format(name)) - cog.out(" {}\n\n".format(doc)) + cog.out(f"``{name}``\n") + cog.out(f" {doc}\n\n") for klass in PAGES.values(): title = "{} page".format(klass.__name__.removesuffix("Context")) - intro = "{} Rendered using the ``{}`` template.".format( - klass.__doc__, klass.documented_template - ) + intro = f"{klass.__doc__} Rendered using the ``{klass.documented_template}`` template." _section(cog, title, intro) if klass.extras_scope is not None: cog.out( @@ -36,10 +34,10 @@ def template_context(cog): "` for this page.\n\n" ) for f in sorted(klass.documented_fields(), key=lambda f: f.name): - cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) - cog.out(" {}\n\n".format(f.help)) + cog.out(f"``{f.name}`` - ``{f.type_name}``\n") + cog.out(f" {f.help}\n\n") def _section(cog, title, intro): cog.out("{}\n{}\n\n".format(title, "-" * len(title))) - cog.out("{}\n\n".format(intro)) + cog.out(f"{intro}\n\n") diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 15891963..81dc90dc 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl ) -This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX `__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. +This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 `__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. This test also uses the `pytest-asyncio `__ package to add support for ``async def`` test functions running under pytest. @@ -57,7 +57,7 @@ Then run the tests using pytest like so:: Setting up a Datasette test instance ------------------------------------ -The above example shows the easiest way to start writing tests against a Datasette instance: +Use :ref:`datasette.client ` to make requests against a test instance. The first request runs startup hooks and launches registered background tasks automatically: .. code-block:: python @@ -71,16 +71,24 @@ The above example shows the easiest way to start writing tests against a Dataset response = await datasette.client.get("/-/plugins.json") assert response.status_code == 200 -Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this: +If your test uses Datasette directly without making a request, call ``await datasette.invoke_startup()`` to initialize the instance and run its startup hooks: .. code-block:: python datasette = Datasette(memory=True) await datasette.invoke_startup() -This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. +This runs the :ref:`plugin_hook_startup` and :ref:`plugin_hook_prepare_jinja2_environment` hooks on the same event loop as your test. It does not launch registered background tasks. -If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +To run tasks registered with :ref:`datasette_add_background_task` without making a request, use ``await datasette.start_background_tasks()``. This runs startup if needed and launches every registered task: + +.. code-block:: python + + datasette = Datasette(memory=True) + await datasette.start_background_tasks() + # Tasks registered by startup() hooks have been launched + +See :ref:`datasette_lifecycle` for the full startup and shutdown sequence. .. _testing_plugins_datasette_fixtures_database: @@ -154,7 +162,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini` Using datasette.client in tests ------------------------------- -The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. +The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. A simple test looks like this: @@ -273,22 +281,22 @@ If you want to create that test database repeatedly for every individual test fu .. _testing_plugins_pytest_httpx: -Testing outbound HTTP calls with pytest-httpx ---------------------------------------------- +Testing outbound HTTP calls with pytest-httpx2 +---------------------------------------------- If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests. -The `pytest-httpx `__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally. +The `pytest-httpx2 `__ package provides a ``httpx2_mock`` fixture, built on `respx `__, for mocking outbound calls made using HTTPX2. -To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture. +Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock. -As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content: +As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content: .. code-block:: python from datasette import hookimpl from datasette.utils.asgi import Response - import httpx + import httpx2 @hookimpl @@ -306,27 +314,18 @@ As an example, here's a very simple plugin which executes an HTTP response and r """) vars = await request.post_vars() url = vars["url"] - return Response.text(httpx.get(url).text) + return Response.text(httpx2.get(url).text) -Here's a test for that plugin that mocks the HTTPX outbound request: +Here's a test for that plugin that mocks the HTTPX2 outbound request: .. code-block:: python from datasette.app import Datasette - import pytest - @pytest.fixture - def non_mocked_hosts(): - # This ensures httpx-mock will not affect Datasette's own - # httpx calls made in the tests by datasette.client: - return ["localhost"] - - - async def test_outbound_http_call(httpx_mock): - httpx_mock.add_response( - url="https://www.example.com/", - text="Hello world", + async def test_outbound_http_call(httpx2_mock): + httpx2_mock.get("https://www.example.com/").respond( + text="Hello world" ) datasette = Datasette([], memory=True) response = await datasette.client.post( @@ -335,11 +334,13 @@ Here's a test for that plugin that mocks the HTTPX outbound request: ) assert response.text == "Hello world" - outbound_request = httpx_mock.get_request() + outbound_request = httpx2_mock.calls.last.request assert ( outbound_request.url == "https://www.example.com/" ) +If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx `__. + .. _testing_plugins_register_in_test: Registering a plugin for the duration of a test diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index d1e5e75a..bbefe344 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -203,6 +203,40 @@ Templates should be bundled for distribution using the same ``package_data`` mec You can also use wildcards here such as ``templates/*.html``. See `datasette-edit-schema `__ for an example of this pattern. +.. _writing_plugins_custom_templates_breadcrumbs: + +Adding breadcrumbs +~~~~~~~~~~~~~~~~~~ + +Plugin templates that extend ``base.html`` can use the ``crumbs.nav()`` macro to display breadcrumb links back to the Datasette homepage, and optionally to a database and a table. Override the ``crumbs`` block to specify which links to include: + +.. code-block:: html+jinja + + {% extends "base.html" %} + + {% block title %}Manage {{ table }}{% endblock %} + + {% block crumbs %} + + {{ crumbs.nav(request=request, database=database, table=table) }} + + {{ crumbs.nav(request=request, database=database) }} + {% endblock %} + + {% block content %} +

      Manage {{ table }}

      + {% endblock %} + +The macro accepts these arguments: + +* ``request``: the current request, used to check the actor's permissions. +* ``database``: an optional database name, as a string +* ``table``: an optional table name, as a string. If you pass ``table``, you must also pass ``database``. + +For a database-level plugin page, use ``{{ crumbs.nav(request=request, database=database) }}``. For a page with just a homepage link, use ``{{ crumbs.nav(request=request) }}``, which is also the default provided by ``base.html`` if you do not override the block. + +The table-level example renders links in the form ``home / database / table``. Each link is only included if the current actor has permission to view that resource. + .. _writing_plugins_configuration: Writing plugins that accept configuration diff --git a/pyproject.toml b/pyproject.toml index 215b2cca..cd8b5513 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,18 +28,19 @@ dependencies = [ "click-default-group>=1.2.3", "Jinja2>=2.10.3", "hupper>=1.9", - "httpx>=0.20,<1.0", + "httpx2>=2.0", "pluggy>=1.0", - "uvicorn>=0.11", + "uvicorn>=0.29", "aiofiles>=0.4", "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", - "sqlite-utils>=3.30,<4.0", + "sqlite-utils>=4.0", "asyncinject>=0.7", "setuptools", "pip", "pydantic>=2", + "opentelemetry-api>=1.37", ] [project.urls] @@ -63,16 +64,17 @@ dev = [ "pytest-xdist>=2.2.1", "pytest-asyncio>=1.2.0", "beautifulsoup4>=4.8.1", - "black==26.3.1", + "black==26.5.1", "blacken-docs==1.20.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", "multipart-form-data-conformance==0.1a0", - "ruff>=0.9", + "ruff>=0.16.0", + "opentelemetry-sdk>=1.37", # docs "Sphinx==7.4.7", - "furo==2025.9.25", + "furo==2025.12.19", "sphinx-autobuild", "codespell>=2.2.5", "sphinx-copybutton", @@ -85,6 +87,9 @@ dev = [ playwright = [ "pytest-playwright>=0.8.0", ] +shots = [ + "shot-scraper>=1.12", +] [project.optional-dependencies] rich = ["rich"] @@ -102,9 +107,5 @@ datasette = ["templates/*.html"] [tool.setuptools.dynamic] version = {attr = "datasette.version.__version__"} -[tool.ruff] -line-length = 160 -select = ["E", "F", "W"] - [tool.uv] package = true diff --git a/pytest.ini b/pytest.ini index 75de6925..590054de 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +addopts = --ignore=ignored filterwarnings= # https://github.com/pallets/jinja/issues/927 ignore:Using or importing the ABCs::jinja2 diff --git a/ruff.toml b/ruff.toml index 74447a8c..3c4345bf 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,2 +1,7 @@ line-length = 160 -target-version = "py310" \ No newline at end of file +target-version = "py310" + +[lint.flake8-bugbear] +# from_extra() returns a dataclasses.field(), so it is safe as a dataclass +# default - ruff cannot see through the wrapper (RUF009) +extend-immutable-calls = ["datasette.views.from_extra"] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 7ec03146..5fdbd51d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,18 @@ -import httpx import importlib.metadata import os import pathlib -import pytest -import pytest_asyncio import re +import socket import subprocess import sys import tempfile import time from dataclasses import dataclass + +import httpx2 +import pytest +import pytest_asyncio + from datasette import Event, hookimpl try: @@ -30,15 +33,39 @@ UNDOCUMENTED_PERMISSIONS = { } -def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): +def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs): start = time.time() while time.time() - start < timeout: + # If the server died there is no point waiting out the timeout - fail + # now, with its output, instead of after `timeout` seconds of silence + if process is not None and process.poll() is not None: + raise AssertionError( + "Server exited early with returncode {}\n{}".format( + process.returncode, process.stdout.read().decode("utf-8") + ) + ) try: client.get(url, **kwargs) return - except httpx.ConnectError: + except httpx2.TransportError: time.sleep(0.1) - raise AssertionError("Timed out waiting for {} to respond".format(url)) + raise AssertionError(f"Timed out waiting for {url} to respond") + + +def find_free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +from datasette.telemetry_testing import ( # noqa: F401 + MetricsCollector, + otel_meter_provider, + otel_metrics, + otel_provider, + otel_reset, + otel_spans, +) @pytest.fixture @@ -54,10 +81,12 @@ def bare_ds(): @pytest_asyncio.fixture(scope="session") async def ds_client(): + import secrets + from datasette.app import Datasette from datasette.database import Database + from .fixtures import CONFIG, METADATA, PLUGINS_DIR - import secrets ds = Datasette( metadata=METADATA, @@ -87,7 +116,10 @@ async def ds_client(): await db.execute_write_fn(prepare) await ds.invoke_startup() - return ds.client + try: + yield ds.client + finally: + ds.close() def pytest_report_header(config): @@ -96,8 +128,8 @@ def pytest_report_header(config): conn.close() sqlite_utils_version = importlib.metadata.version("sqlite-utils") headers = [ - "SQLite: {}".format(version), - "sqlite-utils: {}".format(sqlite_utils_version), + f"SQLite: {version}", + f"sqlite-utils: {sqlite_utils_version}", ] if config.getoption("--playwright"): try: @@ -149,6 +181,11 @@ def pytest_collection_modifyitems(config, items): move_to_front(items, "test_spatialite_error_if_attempt_to_open_spatialite") move_to_front(items, "test_package") move_to_front(items, "test_package_with_port") + # These start subprocesses, which can crash on macOS/CPython 3.13 late in + # a test run once the pytest process has started many threads + move_to_front(items, "test_datasette_package_never_imports_the_sdk") + move_to_front(items, "test_kit_module_itself_never_imports_the_sdk") + move_to_front(items, "test_no_provider_takes_the_fast_path") def move_to_front(items, test_name): @@ -175,8 +212,8 @@ def restore_working_directory(tmpdir, request): @pytest.fixture(scope="session", autouse=True) def check_actions_are_documented(): - from datasette.plugins import pm from datasette.default_actions import register_actions as default_register_actions + from datasette.plugins import pm content = ( pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" @@ -202,7 +239,7 @@ def check_actions_are_documented(): if kwargs["action"] in core_actions: assert ( action in documented_actions - ), "Undocumented permission action: {}".format(action) + ), f"Undocumented permission action: {action}" pm.add_hookcall_monitoring( before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None @@ -247,12 +284,24 @@ def ds_localhost_http_server(): # Avoid FileNotFoundError: [Errno 2] No such file or directory: cwd=tempfile.gettempdir(), ) - wait_until_responds("http://localhost:8041/") - # Check it started successfully - assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") - yield ds_proc - # Shut it down at the end of the pytest session - ds_proc.terminate() + try: + wait_until_responds("http://localhost:8041/", process=ds_proc) + yield ds_proc + finally: + stop_process(ds_proc) + + +def stop_process(proc): + try: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + finally: + proc.stdout.close() @pytest.fixture(scope="session") @@ -273,11 +322,27 @@ def ds_unix_domain_socket_server(tmp_path_factory): cwd=tempfile.gettempdir(), ) # Poll until available - transport = httpx.HTTPTransport(uds=uds) - client = httpx.Client(transport=transport) + transport = httpx2.HTTPTransport(uds=uds) + client = httpx2.Client(transport=transport) try: + # Probe with a socket we own: the HTTP transport can leak a socket + # when connect() fails before the UDS server has started listening. + start = time.monotonic() + while True: + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe: + probe.settimeout(0.1) + probe.connect(uds) + break + except OSError: + if ds_proc.poll() is not None or time.monotonic() - start > 30: + raise + time.sleep(0.1) wait_until_responds( - "http://localhost/_memory.json", timeout=30.0, client=client + "http://localhost/_memory.json", + timeout=30.0, + client=client, + process=ds_proc, ) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") @@ -285,20 +350,75 @@ def ds_unix_domain_socket_server(tmp_path_factory): finally: client.close() # Shut it down at the end of the pytest session - ds_proc.terminate() - try: - ds_proc.wait(timeout=5) - except subprocess.TimeoutExpired: - ds_proc.kill() - ds_proc.wait() + stop_process(ds_proc) try: os.unlink(uds) except FileNotFoundError: pass +@pytest.fixture +def serve_with_plugins(tmp_path): + """Factory fixture for starting ``datasette serve`` in a subprocess with + plugins written to a temporary ``--plugins-dir``. + + For tests that need the real serve path: event-loop wiring, exit codes, + signals. The usual in-process ``pm.register`` plugin pattern can't reach + a subprocess, so plugin source is written out as importable files instead. + + Unlike ``ds_localhost_http_server`` this is function-scoped and takes a + fresh port each time, because each test needs its own plugins. Call it as:: + + proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE}) + + ``plugins`` maps module name to Python source. Pass + ``wait_for_startup=False`` when the server is expected to fail during + startup rather than begin serving. Extra CLI arguments are passed through. + Every process started is terminated when the test ends. + """ + processes = [] + + def start(plugins, *extra_args, wait_for_startup=True): + plugins_dir = tmp_path / "plugins" + plugins_dir.mkdir(exist_ok=True) + for module_name, source in plugins.items(): + (plugins_dir / f"{module_name}.py").write_text(source, "utf-8") + port = find_free_port() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "datasette", + "--memory", + "--plugins-dir", + str(plugins_dir), + "-h", + "127.0.0.1", + "-p", + str(port), + *extra_args, + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + # Avoid FileNotFoundError: [Errno 2] No such file or directory: + cwd=tempfile.gettempdir(), + ) + processes.append(proc) + if wait_for_startup: + wait_until_responds( + f"http://127.0.0.1:{port}/-/versions.json", process=proc + ) + return proc, port + + yield start + + for proc in processes: + stop_process(proc) + + # Import fixtures from fixtures.py to make them available -from .fixtures import ( # noqa: E402, F401 +from .fixtures import ( # noqa: F401 + TEMP_PLUGIN_SECRET_FILE, app_client, app_client_base_url_prefix, app_client_conflicting_database_names, @@ -315,5 +435,4 @@ from .fixtures import ( # noqa: E402, F401 app_client_with_dot, app_client_with_trace, make_app_client, - TEMP_PLUGIN_SECRET_FILE, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index 8ab3633f..f8710ad4 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,3 +1,13 @@ +import contextlib +import json +import os +import pathlib +import tempfile +import textwrap + +import click +import pytest + from datasette.app import Datasette from datasette.fixtures import ( EXTRA_DATABASE_SQL, @@ -5,14 +15,6 @@ from datasette.fixtures import ( write_fixture_database, ) from datasette.utils.testing import TestClient -import click -import contextlib -import json -import os -import pathlib -import pytest -import tempfile -import textwrap # This temp file is used by one of the plugin config tests TEMP_PLUGIN_SECRET_FILE = os.path.join(tempfile.gettempdir(), "plugin-secret") @@ -167,12 +169,10 @@ def make_app_client( template_dir=template_dir, crossdb=crossdb, ) - yield TestClient(ds) - # Close as many database connections as possible - # to try and avoid too many open files error - for db in ds.databases.values(): - if not db.is_memory: - db.close() + try: + yield TestClient(ds) + finally: + ds.close() @pytest.fixture(scope="session") @@ -184,9 +184,10 @@ def app_client(): @pytest.fixture(scope="session") def app_client_no_files(): ds = Datasette([]) - yield TestClient(ds) - for db in ds.databases.values(): - db.close() + try: + yield TestClient(ds) + finally: + ds.close() @pytest.fixture(scope="session") diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index c89ad0a3..baf20a77 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -1,16 +1,16 @@ import asyncio -from datasette import hookimpl -from datasette.facets import Facet -from datasette.tokens import TokenHandler -from datasette import tracer -from datasette.permissions import Action -from datasette.resources import DatabaseResource -from datasette.utils import path_with_added_args -from datasette.utils.asgi import asgi_send_json, Response import base64 import json import urllib.parse +from datasette import hookimpl, tracer +from datasette.facets import Facet +from datasette.permissions import Action +from datasette.resources import DatabaseResource +from datasette.tokens import TokenHandler +from datasette.utils import path_with_added_args +from datasette.utils.asgi import Response, asgi_send_json + @hookimpl def prepare_connection(conn, database, datasette): @@ -305,11 +305,7 @@ def startup(datasette): datasette._startup_hook_fired = True # And test some import shortcuts too - from datasette import Response - from datasette import Forbidden - from datasette import NotFound - from datasette import hookimpl - from datasette import actor_matches_allow + from datasette import Forbidden, NotFound, Response, actor_matches_allow, hookimpl _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow) @@ -373,7 +369,7 @@ def table_actions(datasette, database, table, actor, request): "label": "Plugin button", "description": "Runs JavaScript from a plugin", "attrs": { - "aria-label": "Plugin button for {}".format(table), + "aria-label": f"Plugin button for {table}", "data-plugin-action": "plugin-button", "data-database": database, "data-table": table, diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 864637a6..26e45a5b 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -1,8 +1,10 @@ +import json +from functools import wraps + +import markupsafe + from datasette import hookimpl from datasette.utils.asgi import Response -from functools import wraps -import markupsafe -import json @hookimpl @@ -33,11 +35,7 @@ def render_cell(value, database): if set(data.keys()) != {"href", "label"}: return None href = data["href"] - if not ( - href.startswith("/") - or href.startswith("http://") - or href.startswith("https://") - ): + if not (href.startswith(("/", "http://", "https://"))): return None return markupsafe.Markup( '{label}'.format( @@ -54,7 +52,7 @@ def extra_template_vars(template, database, table, view_name, request, datasette datasette._last_request = request async def query_database(sql): - first_db = list(datasette.databases.keys())[0] + first_db = next(iter(datasette.databases.keys())) return (await datasette.execute(first_db, sql)).rows[0][0] async def inner(): @@ -172,10 +170,10 @@ def register_routes(datasette): path = config["path"] def new_table(request): - return Response.text("/db/table: {}".format(sorted(request.url_vars.items()))) + return Response.text(f"/db/table: {sorted(request.url_vars.items())}") return [ - (r"/{}/$".format(path), lambda: Response.text(path.upper())), + (rf"/{path}/$", lambda: Response.text(path.upper())), # Also serves to demonstrate over-ride of default paths: (r"/(?P[^/]+)/(?P[^/]+?$)", new_table), ] diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py index cfe15215..671f2d1e 100644 --- a/tests/plugins/register_output_renderer.py +++ b/tests/plugins/register_output_renderer.py @@ -1,6 +1,7 @@ +import json + from datasette import hookimpl from datasette.utils.asgi import Response -import json async def can_render( @@ -18,9 +19,7 @@ async def can_render( "request": request, "view_name": view_name, } - if request.args.get("_no_can_render"): - return False - return True + return not request.args.get("_no_can_render") async def render_test_all_parameters( diff --git a/tests/plugins/sleep_sql_function.py b/tests/plugins/sleep_sql_function.py index d4b32a09..2fca1d66 100644 --- a/tests/plugins/sleep_sql_function.py +++ b/tests/plugins/sleep_sql_function.py @@ -1,6 +1,7 @@ -from datasette import hookimpl import time +from datasette import hookimpl + @hookimpl def prepare_connection(conn): diff --git a/tests/test_actions_sql.py b/tests/test_actions_sql.py index a1fca971..76320bd1 100644 --- a/tests/test_actions_sql.py +++ b/tests/test_actions_sql.py @@ -10,10 +10,11 @@ These tests verify: import pytest import pytest_asyncio + +from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.resources import DatabaseResource, QueryResource, TableResource -from datasette import hookimpl def test_resource_string_representations(): @@ -90,7 +91,7 @@ async def test_allowed_resources_global_allow(test_ds): assert all(isinstance(t, TableResource) for t in tables) # Check specific tables are present - table_set = set((t.parent, t.child) for t in tables) + table_set = {(t.parent, t.child) for t in tables} assert ("analytics", "events") in table_set assert ("analytics", "users") in table_set assert ("analytics", "sensitive") in table_set diff --git a/tests/test_actor_restriction_bug.py b/tests/test_actor_restriction_bug.py index 0bfc9e1e..6e633ff6 100644 --- a/tests/test_actor_restriction_bug.py +++ b/tests/test_actor_restriction_bug.py @@ -6,6 +6,7 @@ config allow blocks can bypass table-level restrictions. """ import pytest + from datasette.app import Datasette from datasette.resources import TableResource diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 08b952fb..2f20e8e8 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -10,6 +10,8 @@ Layer 3: table/database views precompute all registered actions before import pytest import pytest_asyncio + +from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import ( Action, @@ -18,7 +20,6 @@ from datasette.permissions import ( _permission_check_cache, ) from datasette.resources import DatabaseResource, TableResource -from datasette import hookimpl class CountingRulesPlugin: @@ -114,7 +115,7 @@ async def test_allowed_not_memoized_without_cache(counting_ds): async def test_cache_keyed_on_full_actor_identity(counting_ds): """Interleaved checks for different actors never share cache entries.""" # Uses drop-table because default permissions deny it to non-root actors - ds, plugin = counting_ds + ds, _plugin = counting_ds resource = TableResource("analytics", "users") token = _permission_check_cache.set({}) try: @@ -180,7 +181,7 @@ async def test_cache_keyed_on_resource(counting_ds): @pytest.mark.asyncio async def test_skip_permission_checks_bypasses_cache(counting_ds): - ds, plugin = counting_ds + ds, _plugin = counting_ds resource = TableResource("analytics", "users") token = _permission_check_cache.set({}) try: diff --git a/tests/test_allowed_resources.py b/tests/test_allowed_resources.py index e247aa78..e251deab 100644 --- a/tests/test_allowed_resources.py +++ b/tests/test_allowed_resources.py @@ -7,9 +7,10 @@ based on permission rules from plugins and configuration. import pytest import pytest_asyncio + +from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import PermissionSQL -from datasette import hookimpl # Test plugin that provides permission rules diff --git a/tests/test_api.py b/tests/test_api.py index f57d0206..690cb080 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,12 +1,16 @@ +import pathlib +import urllib + +import pytest + from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS +from datasette.resources import DatabaseResource, TableResource +from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode from datasette.utils.sqlite import sqlite_version from datasette.version import __version__ -from .fixtures import make_app_client, EXPECTED_PLUGINS -import pathlib -import pytest -import sys -import urllib + +from .fixtures import EXPECTED_PLUGINS, make_app_client @pytest.mark.asyncio @@ -15,7 +19,7 @@ async def test_homepage(ds_client): assert response.status_code == 200 assert "application/json; charset=utf-8" == response.headers["content-type"] data = response.json() - assert sorted(list(data.get("metadata").keys())) == [ + assert sorted(data.get("metadata").keys()) == [ "about", "about_url", "description_html", @@ -26,8 +30,9 @@ async def test_homepage(ds_client): "title", ] databases = data.get("databases") - assert databases.keys() == {"fixtures": 0}.keys() - d = databases["fixtures"] + assert isinstance(databases, list) + assert [d["name"] for d in databases] == ["fixtures"] + d = databases[0] assert d["name"] == "fixtures" assert isinstance(d["tables_count"], int) assert isinstance(len(d["tables_and_views_truncated"]), int) @@ -42,8 +47,7 @@ async def test_homepage_sort_by_relationships(ds_client): response = await ds_client.get("/.json?_sort=relationships") assert response.status_code == 200 tables = [ - t["name"] - for t in response.json()["databases"]["fixtures"]["tables_and_views_truncated"] + t["name"] for t in response.json()["databases"][0]["tables_and_views_truncated"] ] assert tables == [ "simple_primary_key", @@ -98,14 +102,11 @@ async def test_database_page(ds_client): "tags", } - # Expected hidden tables + # The external-content index is visible, but its shadow tables need a + # second dependency hop and are excluded by the one-hop permission policy. expected_hidden_tables = { "no_primary_key", "searchable_fts", - "searchable_fts_config", - "searchable_fts_data", - "searchable_fts_docsize", - "searchable_fts_idx", } # Verify all expected tables exist @@ -250,8 +251,10 @@ def test_no_files_uses_memory_database(app_client_no_files): response = app_client_no_files.get("/.json") assert response.status == 200 assert { - "databases": { - "_memory": { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "databases": [ + { "name": "_memory", "hash": None, "color": "a6c7b9", @@ -266,7 +269,7 @@ def test_no_files_uses_memory_database(app_client_no_files): "views_count": 0, "private": False, }, - }, + ], "metadata": {}, } == response.json # Try that SQL query @@ -323,20 +326,15 @@ def test_sql_time_limit(app_client_shorter_time_limit): "/fixtures/-/query.json?sql=select+sleep(0.5)", ) assert 400 == response.status + expected_message = ( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ) assert response.json == { "ok": False, - "error": ( - "

      SQL query took too long. The time limit is controlled by the\n" - 'sql_time_limit_ms\n' - "configuration option.

      \n" - '\n' - "" - ), + "error": expected_message, + "errors": [expected_message], "status": 400, - "title": "SQL Interrupted", } @@ -350,7 +348,7 @@ async def test_custom_sql_time_limit(ds_client): "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5", ) assert response.status_code == 400 - assert response.json()["title"] == "SQL Interrupted" + assert response.json()["error"].startswith("SQL query took too long.") @pytest.mark.asyncio @@ -371,6 +369,38 @@ async def test_row(ds_client): assert response.json()["rows"] == [{"id": 1, "content": "hello"}] +@pytest.mark.asyncio +@pytest.mark.parametrize("suffix", ("", ".json")) +@pytest.mark.parametrize( + "row_path", + ( + "a", # too few components for a two-column primary key + "a,b,c", # too many components for a two-column primary key + ), +) +async def test_row_pk_arity_mismatch_returns_400(ds_client, row_path, suffix): + # A row URL with the wrong number of comma-separated primary key + # components used to raise an uncaught sqlite3.ProgrammingError (HTTP 500) + # because the SQL had one bind placeholder per PK column but params were + # only bound for the supplied components. It should be a 400 instead, + # mirroring the existing guard in datasette/views/table.py. + response = await ds_client.get(f"/fixtures/compound_primary_key/{row_path}{suffix}") + assert response.status_code == 400 + if suffix == ".json": + assert response.json()["ok"] is False + assert response.json()["status"] == 400 + + +@pytest.mark.asyncio +async def test_row_compound_pk_correct_arity(ds_client): + # The valid two-component URL still resolves the row. + response = await ds_client.get( + "/fixtures/compound_primary_key/a,b.json?_shape=objects" + ) + assert response.status_code == 200 + assert response.json()["rows"] == [{"pk1": "a", "pk2": "b", "content": "c"}] + + @pytest.mark.asyncio async def test_row_strange_table_name(ds_client): response = await ds_client.get( @@ -426,10 +456,71 @@ async def test_row_foreign_key_tables(ds_client): ] +@pytest.mark.asyncio +async def test_row_foreign_key_tables_omit_denied_tables(request): + actor = {"id": "reader"} + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "tables": { + "parents": {"permissions": {"view-table": True}}, + "private_children": {"permissions": {"view-table": False}}, + } + } + } + }, + ) + request.addfinalizer(ds.close) + db = ds.add_memory_database("fk_count_leak", name="data") + await db.execute_write("create table parents (id integer primary key, name text)") + await db.execute_write(""" + create table private_children ( + id integer primary key, + parent_id integer references parents(id) + ) + """) + await db.execute_write("insert into parents values (1, 'Public parent')") + await db.execute_write(""" + insert into private_children (id, parent_id) values + (1, 1), + (2, 1), + (3, 1) + """) + await ds.invoke_startup() + + parent = TableResource(database="data", table="parents") + private_children = TableResource(database="data", table="private_children") + assert await ds.allowed(action="view-table", resource=parent, actor=actor) + assert not await ds.allowed( + action="view-table", resource=private_children, actor=actor + ) + assert not await ds.allowed( + action="execute-sql", + resource=DatabaseResource(database="data"), + actor=actor, + ) + + direct_child = await ds.client.get("/data/private_children.json", actor=actor) + assert direct_child.status_code == 403 + parent_response = await ds.client.get( + "/data/parents/1.json?_extra=foreign_key_tables", actor=actor + ) + assert parent_response.status_code == 200 + + foreign_key_tables = parent_response.json().get("foreign_key_tables", []) + assert foreign_key_tables == [], ( + "denied child table name, foreign-key column, and row count disclosed: " + f"{foreign_key_tables}" + ) + + @pytest.mark.asyncio async def test_row_extras(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables,column_details" ) assert response.status_code == 200 data = response.json() @@ -446,6 +537,45 @@ async def test_row_extras(ds_client): "format": "json", } assert len(data["foreign_key_tables"]) == 5 + id_detail = data["column_details"]["id"] + assert id_detail["type"].lower() == "integer" + assert id_detail == { + "type": id_detail["type"], + "sqlite_type": "INTEGER", + "notnull": False, + "default": None, + "is_pk": True, + "pk_position": 1, + "hidden": 0, + } + content_detail = data["column_details"]["content"] + assert content_detail["type"].lower() == "text" + assert content_detail == { + "type": content_detail["type"], + "sqlite_type": "TEXT", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, + } + + +@pytest.mark.asyncio +async def test_column_details_extra_row_for_null_blob(ds_client): + response = await ds_client.get("/fixtures/binary_data/3.json?_extra=column_details") + assert response.status_code == 200 + data_detail = response.json()["column_details"]["data"] + assert data_detail["type"].lower() == "blob" + assert data_detail == { + "type": data_detail["type"], + "sqlite_type": "BLOB", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, + } @pytest.mark.asyncio @@ -507,7 +637,7 @@ async def test_row_extra_render_cell(): def test_databases_json(app_client_two_attached_databases_one_immutable): response = app_client_two_attached_databases_one_immutable.get("/-/databases.json") - databases = response.json + databases = response.json["databases"] assert 2 == len(databases) extra_database, fixtures_database = databases assert "extra database" == extra_database["name"] @@ -523,10 +653,13 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): @pytest.mark.asyncio async def test_threads_json(ds_client): - response = await ds_client.get("/-/threads.json") - expected_keys = {"threads", "num_threads"} - if sys.version_info >= (3, 7, 0): - expected_keys.update({"tasks", "num_tasks"}) + ds_client.ds.root_enabled = True + try: + response = await ds_client.get("/-/threads.json", actor={"id": "root"}) + finally: + ds_client.ds.root_enabled = False + expected_keys = {"ok", "threads", "num_threads"} + expected_keys.update({"tasks", "num_tasks"}) data = response.json() assert set(data.keys()) == expected_keys # Should be at least one _execute_writes thread for __INTERNAL__ @@ -577,7 +710,7 @@ async def test_actions_json(ds_client): try: ds_client.ds.root_enabled = True response = await ds_client.get("/-/actions.json", actor={"id": "root"}) - data = response.json() + data = response.json()["actions"] finally: ds_client.ds.root_enabled = original_root_enabled assert isinstance(data, list) @@ -609,6 +742,7 @@ async def test_actions_json(ds_client): async def test_settings_json(ds_client): response = await ds_client.get("/-/settings.json") assert response.json() == { + "ok": True, "default_page_size": 50, "default_facet_size": 30, "default_allow_sql": True, @@ -616,6 +750,7 @@ async def test_settings_json(ds_client): "facet_time_limit_ms": 200, "max_returned_rows": 100, "max_insert_rows": 100, + "max_post_body_bytes": 2 * 1024 * 1024, "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, @@ -677,7 +812,7 @@ def test_config_cache_size(app_client_larger_cache_size): def test_config_force_https_urls(): with make_app_client(settings={"force_https_urls": True}) as client: response = client.get( - "/fixtures/facetable.json?_size=3&_facet=state&_extra=next_url,suggested_facets" + "/fixtures/facetable.json?_size=3&_facet=state&_extra=suggested_facets" ) assert response.json["next_url"].startswith("https://") assert response.json["facet_results"]["results"]["state"]["results"][0][ @@ -772,7 +907,9 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 assert ["foo-bar", "foo", "fixtures"] == [ d["name"] - for d in app_client_conflicting_database_names.get("/-/databases.json").json + for d in app_client_conflicting_database_names.get("/-/databases.json").json[ + "databases" + ] ] for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): data = app_client_conflicting_database_names.get(path).json @@ -816,10 +953,7 @@ async def test_hidden_sqlite_stat1_table(): await db.execute_write("analyze") data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] - assert tables in ( - [("normal", False), ("sqlite_stat1", True)], - [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)], - ) + assert tables == [("normal", False)] @pytest.mark.asyncio @@ -843,13 +977,41 @@ async def test_tilde_encoded_database_names(db_name): ds = Datasette() ds.add_memory_database(db_name) response = await ds.client.get("/.json") - assert db_name in response.json()["databases"].keys() - path = response.json()["databases"][db_name]["path"] + databases_by_name = {d["name"]: d for d in response.json()["databases"]} + assert db_name in databases_by_name + path = databases_by_name[db_name]["path"] # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 +@pytest.mark.asyncio +@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar")) +async def test_table_with_reserved_characters_in_name(table_name): + # Table names containing characters such as "]" that cannot be escaped + # using SQLite [bracket] quoting used to break schema introspection and + # the table page - https://github.com/simonw/datasette/issues/2431 + ds = Datasette() + db = ds.add_memory_database("test_reserved_table_names") + await db.execute_write( + f"create table {escape_sqlite(table_name)} (id integer primary key, name text)" + ) + await db.execute_write( + f"insert into {escape_sqlite(table_name)} (id, name) values (1, 'one')" + ) + # Schema introspection (populate_schema_tables) must not crash: + db_response = await ds.client.get("/test_reserved_table_names.json") + assert db_response.status_code == 200 + tables = {t["name"]: t for t in db_response.json()["tables"]} + assert tables[table_name]["count"] == 1 + # And the table page itself must load and return the row: + table_response = await ds.client.get( + f"/test_reserved_table_names/{tilde_encode(table_name)}.json?_shape=array" + ) + assert table_response.status_code == 200 + assert table_response.json() == [{"id": 1, "name": "one"}] + + @pytest.mark.asyncio @pytest.mark.parametrize( "config,expected", @@ -883,7 +1045,7 @@ async def test_config_json(config, expected): "/-/config.json should return redacted configuration" ds = Datasette(config=config) response = await ds.client.get("/-/config.json") - assert response.json() == expected + assert response.json() == {"ok": True, **expected} @pytest.mark.asyncio @@ -979,7 +1141,7 @@ async def test_config_json(config, expected): async def test_upgrade_metadata(metadata, expected_config, expected_metadata): ds = Datasette(metadata=metadata) response = await ds.client.get("/-/config.json") - assert response.json() == expected_config + assert response.json() == {"ok": True, **expected_config} response2 = await ds.client.get("/-/metadata.json") assert response2.json() == expected_metadata diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 563ca21e..50bf1d67 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,9 +1,25 @@ +import time + +import pytest +import sqlite_utils + from datasette.app import Datasette from datasette.events import RenameTableEvent -from datasette.utils import escape_sqlite, sqlite3 +from datasette.utils import error_body, escape_sqlite, sqlite3 + from .utils import last_event -import pytest -import time + + +def assert_schema_contains(fragment, schema): + assert ( + fragment in schema + ), f"Expected schema to contain {fragment!r}, got {schema!r}" + + +def assert_schema_not_contains(fragment, schema): + assert ( + fragment not in schema + ), f"Expected schema not to contain {fragment!r}, got {schema!r}" @pytest.fixture @@ -35,21 +51,324 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": "application/json", } +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ["read", "read_row", "rename"]) +async def test_trailing_lf_table_permissions(tmp_path, operation): + # SQLite treats "secret" and "secret\n" as different table names. Permission + # checks and SQL execution must agree on which table a request targets. + db_path = tmp_path / "data.db" + conn = sqlite3.connect(str(db_path)) + conn.executescript( + "create table secret (id integer primary key, value text);" + "insert into secret values (1, 'private');" + ) + conn.close() + # Allow builder to create and use tables generally, but explicitly deny + # access to the existing secret table below. Disable arbitrary SQL access. + grants = { + action: {"id": "builder"} + for action in ( + "view-database", + "create-table", + "view-table", + "insert-row", + "alter-table", + ) + } + ds = Datasette( + [str(db_path)], + default_deny=True, + settings={"default_allow_sql": False}, + config={ + "permissions": {"view-instance": {"id": "builder"}}, + "databases": { + "data": { + "permissions": grants, + "tables": { + "secret": { + "permissions": { + "view-table": False, + "insert-row": False, + "alter-table": False, + } + } + }, + } + }, + }, + ) + headers = _headers(write_token(ds, actor_id="builder")) + try: + # Establish that the protected table is inaccessible before creating + # a second table whose name differs only by a trailing line feed. + response = await ds.client.get("/data/secret.json", headers=headers) + assert response.status_code == 403 + response = await ds.client.get( + "/data/-/query.json?sql=select+*+from+secret", headers=headers + ) + assert response.status_code == 403 + # Distinct values let us detect if an operation targets secret + # instead of the newly created secret\n table. + response = await ds.client.post( + "/data/-/create", + json={"table": "secret\n", "row": {"id": 1, "value": "decoy"}, "pk": "id"}, + headers=headers, + ) + assert response.status_code == 201, response.text + # ~0A is Datasette's URL encoding for the line feed in the table name. + if operation in ("read", "read_row"): + # Both table and row endpoints must return only the permitted row. + path = "/1.json" if operation == "read_row" else ".json" + response = await ds.client.get( + "/data/secret~0A" + path + "?_shape=array", headers=headers + ) + assert response.status_code == 200, response.text + assert response.json() == [{"id": 1, "value": "decoy"}] + else: + # Renaming must move the permitted table, preserving its contents + # and removing its old name from the database. + response = await ds.client.post( + "/data/secret~0A/-/alter", + json={"operations": [{"op": "rename_table", "args": {"to": "moved"}}]}, + headers=headers, + ) + assert response.status_code == 200, response.text + db = ds.get_database("data") + assert ( + await db.execute('select value from "moved"') + ).single_value() == "decoy" + assert "secret\n" not in await db.table_names() + # Verify that the protected table and its data are unchanged, and that + # the API still denies access to it. + db = ds.get_database("data") + assert ( + await db.execute('select value from "secret"') + ).single_value() == "private" + response = await ds.client.get("/data/secret.json", headers=headers) + assert response.status_code == 403 + finally: + ds.close() + + def _insert_and_fetch_created(conn, table, insert_sql): cursor = conn.execute(insert_sql) return conn.execute( - "select created, typeof(created) from {} where rowid = ?".format( - escape_sqlite(table) - ), + f"select created, typeof(created) from {escape_sqlite(table)} where rowid = ?", (cursor.lastrowid,), ).fetchone() +BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"} +BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}' + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_fallback", (False, True)) +@pytest.mark.parametrize( + "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase") +) +@pytest.mark.parametrize( + "module,definition,values,shadow_suffix", + ( + ("fts5", "body", "'original'", "_content"), + ("fts4", "body", "'original'", "_content"), + ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"), + ), +) +@pytest.mark.parametrize("shadow", (False, True)) +async def test_structured_writes_require_ordinary_tables( + ds_write, + monkeypatch, + use_fallback, + operation, + module, + definition, + values, + shadow_suffix, + shadow, +): + if use_fallback: + monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False) + db = ds_write.get_database("data") + await db.execute_write(f"create virtual table indexed using {module}({definition})") + await db.execute_write(f"insert into indexed values ({values})") + table = "indexed" + (shadow_suffix if shadow else "") + row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0] + pks = await db.primary_keys(table) + pk_value = row[pks[0] if pks else "rowid"] + before = await db.execute_fn(lambda conn: list(conn.iterdump())) + + if operation in ("create", "create_uppercase"): + path = "/data/-/create" + body = { + "table": table.upper() if operation == "create_uppercase" else table, + "rows": [row], + } + elif operation in ("update", "delete"): + path = f"/data/{table}/{pk_value}/-/{operation}" + body = {"update": row} if operation == "update" else {} + else: + path = f"/data/{table}/-/{operation}" + body = {"rows": [row]} + response = await ds_write.client.post( + path, json=body, headers=_headers(write_token(ds_write)) + ) + assert response.status_code == 400, response.text + assert response.json()["errors"] == ["Structured writes require an ordinary table"] + assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before + + +@pytest.mark.asyncio +async def test_structured_writes_to_content_table_maintain_fts(ds_write): + db = ds_write.get_database("data") + await db.execute_write_fn( + lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts( + ["title"], create_triggers=True + ) + ) + response = await ds_write.client.post( + "/data/docs/-/insert", + json={"row": {"id": 1, "title": "ordinary content"}}, + headers=_headers(write_token(ds_write)), + ) + assert response.status_code == 201, response.text + matches = await db.execute( + "select rowid from docs_fts where docs_fts match ?", ["ordinary"] + ) + assert [row[0] for row in matches.rows] == [1] + + +@pytest.mark.asyncio +async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "binary_create", + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + "double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert_schema_contains('"data" BLOB', response.json()["schema"]) + assert_schema_contains('"literal" TEXT', response.json()["schema"]) + + rows = (await ds_write.get_database("data").execute(""" + select + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal, + typeof(double_raw) as double_raw_type, + double_raw + from binary_create + """)).dicts() + assert rows == [ + { + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + "double_raw_type": "text", + "double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + } + ] + + +@pytest.mark.asyncio +async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write): + token = write_token(ds_write) + db = ds_write.get_database("data") + await db.execute_write( + "create table binary_api (id integer primary key, data blob, literal text)" + ) + + insert_response = await ds_write.client.post( + "/data/binary_api/-/insert", + json={ + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + }, + headers=_headers(token), + ) + assert insert_response.status_code == 201 + assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE + + upsert_response = await ds_write.client.post( + "/data/binary_api/-/upsert", + json={ + "rows": [ + { + "id": 2, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + ] + }, + headers=_headers(token), + ) + assert upsert_response.status_code == 200 + assert upsert_response.json() == {"ok": True} + + update_response = await ds_write.client.post( + "/data/binary_api/1/-/update", + json={ + "update": { + "data": {"$base64": True, "encoded": "/wAB"}, + "literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "return": True, + }, + headers=_headers(token), + ) + assert update_response.status_code == 200 + assert update_response.json()["rows"][0]["data"] == { + "$base64": True, + "encoded": "/wAB", + } + + rows = (await db.execute(""" + select + id, + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal + from binary_api + order by id + """)).dicts() + assert rows == [ + { + "id": 1, + "data_type": "blob", + "data_hex": "FF0001", + "literal_type": "text", + "literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + }, + { + "id": 2, + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + }, + ] + + @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) @@ -99,7 +418,7 @@ async def test_insert_row(ds_write, content_type): "/data/docs/-/insert", json={"row": {"title": "Test", "score": 1.2, "age": 5}}, headers={ - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": content_type, }, ) @@ -144,11 +463,7 @@ async def test_insert_row_alter(ds_write): @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): token = write_token(ds_write) - data = { - "rows": [ - {"title": "Test {}".format(i), "score": 1.0, "age": 5} for i in range(20) - ] - } + data = {"rows": [{"title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20)]} if return_rows: data["return"] = True response = await ds_write.client.post( @@ -172,14 +487,41 @@ async def test_insert_rows(ds_write, return_rows): ).dicts() assert len(actual_rows) == 20 assert actual_rows == [ - {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} - for i in range(20) + {"id": i + 1, "title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20) ] assert response.json()["ok"] is True if return_rows: assert response.json()["rows"] == actual_rows +@pytest.mark.asyncio +async def test_insert_rows_post_body_too_large(tmp_path_factory): + db_path = str(tmp_path_factory.mktemp("dbs") / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette([db_path], settings={"max_post_body_bytes": 100}) + ds.root_enabled = True + token = write_token(ds) + response = await ds.client.post( + "/data/docs/-/insert", + json={"rows": [{"title": "x" * 200}]}, + headers=_headers(token), + ) + assert response.status_code == 413 + assert response.json() == error_body( + ["Request body exceeded maximum size of 100 bytes"], 413 + ) + # A small body should still work + response2 = await ds.client.post( + "/data/docs/-/insert", + json={"row": {"title": "hi"}}, + headers=_headers(token), + ) + assert response2.status_code == 201 + ds.close() + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,input,special_case,expected_status,expected_errors", @@ -202,8 +544,8 @@ async def test_insert_rows(ds_write, return_rows): "/data/docs/-/insert", {"rows": [{"title": "Test"} for i in range(10)]}, "bad_token", - 403, - ["Permission denied"], + 401, + ["Invalid token signature"], ), ( "/data/docs/-/insert", @@ -214,13 +556,6 @@ async def test_insert_rows(ds_write, return_rows): "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ], ), - ( - "/data/docs/-/insert", - {}, - "invalid_content_type", - 400, - ["Invalid content-type, must be application/json"], - ), ( "/data/docs/-/insert", [], @@ -398,24 +733,21 @@ async def test_insert_or_upsert_row_errors( ) if special_case == "bad_token": token += "bad" - kwargs = dict( - json=input, - headers={ - "Authorization": "Bearer {}".format(token), - "Content-Type": ( - "text/plain" - if special_case == "invalid_content_type" - else "application/json" - ), + kwargs = { + "json": input, + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", }, - ) + } - actor_response = ( - await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) - ).json() - assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( - token_permissions - ) + if special_case != "bad_token": + actor_response = ( + await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) + ).json() + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -462,7 +794,7 @@ async def test_upsert_permissions_per_table(ds_write, allowed): "/data/docs/-/upsert", json={"rows": [{"id": 1, "title": "One"}]}, headers={ - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", }, ) if allowed: @@ -699,9 +1031,7 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Should be a single row assert ( await ds_write.client.get( - "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( - table - ) + f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}" ) ).json() == [1] # Now delete the row @@ -709,14 +1039,12 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Special case for that rowid table delete_path = ( await ds_write.client.get( - "/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}".format( - table - ) + f"/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{table}" ) ).json()[0] delete_response = await ds_write.client.post( - "/data/{}/{}/-/delete".format(table, delete_path), + f"/data/{table}/{delete_path}/-/delete", headers=_headers(write_token(ds_write)), ) assert delete_response.status_code == 200 @@ -729,9 +1057,7 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( - "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( - table - ) + f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}" ) ).json() == [0] @@ -781,21 +1107,26 @@ async def test_update_row_invalid_key(ds_write): pk = await _insert_row(ds_write) - path = "/data/docs/{}/-/update".format(pk) + path = f"/data/docs/{pk}/-/update" response = await ds_write.client.post( path, json={"update": {"title": "New title"}, "bad_key": 1}, headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == {"ok": False, "errors": ["Invalid keys: bad_key"]} + assert response.json() == { + "ok": False, + "error": "Invalid keys: bad_key", + "errors": ["Invalid keys: bad_key"], + "status": 400, + } @pytest.mark.asyncio async def test_update_row_alter(ds_write): token = write_token(ds_write, permissions=["ur", "at"]) pk = await _insert_row(ds_write) - path = "/data/docs/{}/-/update".format(pk) + path = f"/data/docs/{pk}/-/update" response = await ds_write.client.post( path, json={"update": {"title": "New title", "extra": "extra"}, "alter": True}, @@ -909,43 +1240,6 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] -@pytest.mark.asyncio -@pytest.mark.parametrize( - "table,pk_columns", - ( - ("alter_single_pk", ["id"]), - ("alter_compound_pk", ["tenant", "id"]), - ), -) -async def test_alter_table_primary_keys_are_not_null(ds_write, table, pk_columns): - token = write_token(ds_write, permissions=["at"]) - db = ds_write.get_database("data") - await db.execute_write( - "create table {} (id integer, tenant text, title text)".format( - escape_sqlite(table) - ) - ) - - response = await ds_write.client.post( - "/data/{}/-/alter".format(table), - json={ - "operations": [ - {"op": "set_primary_key", "args": {"columns": pk_columns}}, - ] - }, - headers=_headers(token), - ) - - assert response.status_code == 200, response.text - columns = ( - await db.execute( - "select * from pragma_table_info(?) where pk > 0 order by pk", [table] - ) - ).dicts() - assert [column["name"] for column in columns] == pk_columns - assert [column["notnull"] for column in columns] == [1] * len(pk_columns) - - @pytest.mark.asyncio @pytest.mark.parametrize( "default_expr,minimum_value,expected_schema", @@ -988,9 +1282,9 @@ async def test_alter_table_integer_default_expr( assert expected_schema in data["schema"] columns = await db.execute("select * from pragma_table_info('docs')") - created_column = [ + created_column = next( column for column in columns.dicts() if column["name"] == "created" - ][0] + ) assert created_column["type"] == "INTEGER" assert expected_schema in created_column["dflt_value"] @@ -1078,7 +1372,9 @@ async def test_alter_table_foreign_key_operations(ds_write): assert response.status_code == 200, response.text data = response.json() assert data["operations_applied"] == 2 - assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + assert_schema_contains( + '"owner_id" INTEGER REFERENCES "owners"("id")', data["schema"] + ) response = await ds_write.client.post( "/data/docs/-/alter", @@ -1089,7 +1385,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + assert_schema_not_contains('"owner_id" INTEGER REFERENCES', data["schema"]) response = await ds_write.client.post( "/data/docs/-/alter", @@ -1113,7 +1409,9 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert "[owner_id] INTEGER REFERENCES [categories]([id])" in data["schema"] + assert_schema_contains( + '"owner_id" INTEGER REFERENCES "categories"("id")', data["schema"] + ) response = await ds_write.client.post( "/data/docs/-/alter", @@ -1122,7 +1420,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + assert_schema_not_contains('"owner_id" INTEGER REFERENCES', data["schema"]) @pytest.mark.asyncio @@ -1140,10 +1438,9 @@ async def test_alter_table_foreign_key_requires_fk_table_for_fk_column(ds_write) headers=_headers(write_token(ds_write, permissions=["at"])), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["operations.0.add_foreign_key.args: fk_column requires fk_table"], - } + assert response.json() == error_body( + ["operations.0.add_foreign_key.args: fk_column requires fk_table"], 400 + ) @pytest.mark.asyncio @@ -1167,15 +1464,14 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["Could not detect single primary key for table 'accounts'"], - } + assert response.json() == error_body( + ["Could not detect single primary key for table 'accounts'"], 400 + ) @pytest.mark.asyncio async def test_foreign_key_suggestions(ds_write): - token = write_token(ds_write, permissions=["at"]) + token = write_token(ds_write, permissions=["alter-table", "view-table"]) db = ds_write.get_database("data") await db.execute_write("create table owners (id integer primary key)") await db.execute_write("insert into owners (id) values (1), (2), (3)") @@ -1236,15 +1532,12 @@ async def test_foreign_key_suggestions_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == { - "ok": False, - "errors": ["Permission denied: need alter-table"], - } + assert response.json() == error_body(["Permission denied: need alter-table"], 403) @pytest.mark.asyncio async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): - token = write_token(ds_write, permissions=["at"]) + token = write_token(ds_write, permissions=["alter-table", "view-table"]) db = ds_write.get_database("data") await db.execute_write("create table owners (id integer primary key)") @@ -1275,7 +1568,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): @pytest.mark.asyncio async def test_foreign_key_targets(ds_write): - token = write_token(ds_write, permissions=["ct"]) + token = write_token(ds_write, permissions=["create-table", "view-table"]) db = ds_write.get_database("data") await db.execute_write("create table owners (id integer primary key)") await db.execute_write("create table categories (slug varchar(30) primary key)") @@ -1292,7 +1585,8 @@ async def test_foreign_key_targets(ds_write): await db.execute_write("create table no_pk (name text)") try: await db.execute_write("create virtual table search_docs using fts5(body)") - except Exception: + except Exception: # noqa: BLE001, S110 + # FTS5 is not available in every SQLite build pass response = await ds_write.client.get( @@ -1350,10 +1644,7 @@ async def test_foreign_key_targets_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == { - "ok": False, - "errors": ["Permission denied: need create-table"], - } + assert response.json() == error_body(["Permission denied: need create-table"], 403) @pytest.mark.asyncio @@ -1376,10 +1667,7 @@ async def test_alter_table_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == { - "ok": False, - "errors": ["Permission denied: need alter-table"], - } + assert response.json() == error_body(["Permission denied: need alter-table"], 403) @pytest.mark.asyncio @@ -1504,7 +1792,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return): token = write_token(ds_write) pk = await _insert_row(ds_write) - path = "/data/docs/{}/-/update".format(pk) + path = f"/data/docs/{pk}/-/update" data = {"update": input} if use_return: @@ -1523,9 +1811,9 @@ async def test_update_row(ds_write, input, expected_errors, use_return): assert response.json()["ok"] is True if not use_return: - assert "row" not in response.json() + assert "rows" not in response.json() else: - returned_row = response.json()["row"] + returned_row = response.json()["rows"][0] assert returned_row["id"] == pk for k, v in input.items(): assert returned_row[k] == v @@ -1539,7 +1827,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return): # And fetch the row to check it's updated response = await ds_write.client.get( - "/data/docs/{}.json?_shape=array".format(pk), + f"/data/docs/{pk}.json?_shape=array", ) assert response.status_code == 200 row = response.json()[0] @@ -1613,6 +1901,42 @@ async def test_drop_table(ds_write, scenario): assert (await ds_write.client.get("/data/docs")).status_code == 404 +@pytest.mark.asyncio +async def test_drop_table_cleans_up_fts(ds_write): + db = ds_write.get_database("data") + + def enable_fts(conn): + sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True) + + await db.execute_write_fn(enable_fts) + assert { + row[0] + for row in await db.execute( + "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" + ) + } == { + "docs_fts", + "docs_fts_config", + "docs_fts_data", + "docs_fts_docsize", + "docs_fts_idx", + } + + response = await ds_write.client.post( + "/data/docs/-/drop", + json={"confirm": True}, + headers=_headers(write_token(ds_write)), + ) + + assert response.json() == {"ok": True} + assert [ + row[0] + for row in await db.execute( + "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" + ) + ] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_status,expected_response,expected_events", @@ -1660,12 +1984,12 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/one", "table_api_url": "http://localhost/data/one.json", "schema": ( - "CREATE TABLE [one] (\n" - " [id] INTEGER PRIMARY KEY NOT NULL,\n" - " [title] TEXT,\n" - " [score] INTEGER,\n" - " [weight] FLOAT,\n" - " [thumbnail] BLOB\n" + 'CREATE TABLE "one" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "score" INTEGER,\n' + ' "weight" REAL,\n' + ' "thumbnail" BLOB\n' ")" ), }, @@ -1697,10 +2021,10 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/two", "table_api_url": "http://localhost/data/two.json", "schema": ( - "CREATE TABLE [two] (\n" - " [id] INTEGER PRIMARY KEY NOT NULL,\n" - " [title] TEXT,\n" - " [score] FLOAT\n" + 'CREATE TABLE "two" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "score" REAL\n' ")" ), "row_count": 2, @@ -1726,10 +2050,10 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/three", "table_api_url": "http://localhost/data/three.json", "schema": ( - "CREATE TABLE [three] (\n" - " [id] INTEGER PRIMARY KEY NOT NULL,\n" - " [title] TEXT,\n" - " [score] FLOAT\n" + 'CREATE TABLE "three" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "score" REAL\n' ")" ), "row_count": 1, @@ -1751,7 +2075,7 @@ async def test_drop_table(ds_write, scenario): "table": "four", "table_url": "http://localhost/data/four", "table_api_url": "http://localhost/data/four.json", - "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), + "schema": ('CREATE TABLE "four" (\n' ' "name" TEXT\n' ")"), "row_count": 1, }, ["create-table", "insert-rows"], @@ -1771,9 +2095,8 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/five", "table_api_url": "http://localhost/data/five.json", "schema": ( - "CREATE TABLE [five] (\n [type] TEXT NOT NULL,\n" - " [key] INTEGER NOT NULL,\n [title] TEXT,\n" - " PRIMARY KEY ([type], [key])\n)" + 'CREATE TABLE "five" (\n "type" TEXT,\n "key" INTEGER,\n' + ' "title" TEXT,\n PRIMARY KEY ("type", "key")\n)' ), "row_count": 1, }, @@ -2059,87 +2382,18 @@ async def test_create_table( ) assert response.status_code == expected_status data = response.json() + if expected_response.get("ok") is False: + # Error expectations list their messages; derive the canonical envelope + expected_response = error_body(expected_response["errors"], expected_status) + if isinstance(expected_response, dict) and "schema" in expected_response: + assert data.get("schema") == expected_response["schema"] + expected_response = dict(expected_response, schema=data.get("schema")) assert data == expected_response # Should have tracked the expected events events = ds_write._tracked_events assert [e.name for e in events] == expected_events -@pytest.mark.asyncio -@pytest.mark.parametrize( - "body,pk_columns", - ( - ( - { - "table": "pk_from_columns", - "columns": [ - {"name": "id", "type": "integer"}, - {"name": "title", "type": "text"}, - ], - "pk": "id", - }, - ["id"], - ), - ( - { - "table": "compound_pk_from_columns", - "columns": [ - {"name": "tenant", "type": "text"}, - {"name": "id", "type": "integer"}, - {"name": "title", "type": "text"}, - ], - "pks": ["tenant", "id"], - }, - ["tenant", "id"], - ), - ( - { - "table": "pk_omitted_from_columns", - "columns": [ - {"name": "title", "type": "text"}, - ], - "pk": "id", - }, - ["id"], - ), - ( - { - "table": "pk_from_rows", - "rows": [{"id": 1, "title": "Row 1"}], - "pk": "id", - }, - ["id"], - ), - ( - { - "table": "compound_pk_from_rows", - "row": {"tenant": "datasette", "id": 1, "title": "Row 1"}, - "pks": ["tenant", "id"], - }, - ["tenant", "id"], - ), - ), -) -async def test_create_table_primary_keys_are_not_null(ds_write, body, pk_columns): - token = write_token(ds_write) - response = await ds_write.client.post( - "/data/-/create", - json=body, - headers=_headers(token), - ) - - assert response.status_code == 201, response.text - db = ds_write.get_database("data") - columns = ( - await db.execute( - "select * from pragma_table_info(?) where pk > 0 order by pk", - [body["table"]], - ) - ).dicts() - assert [column["name"] for column in columns] == pk_columns - assert [column["notnull"] for column in columns] == [1] * len(pk_columns) - - @pytest.mark.asyncio async def test_create_table_with_foreign_key(ds_write): token = write_token(ds_write) @@ -2176,7 +2430,9 @@ async def test_create_table_with_foreign_key(ds_write): ) assert response.status_code == 201 data = response.json() - assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + assert_schema_contains( + '"owner_id" INTEGER REFERENCES "owners"("id")', data["schema"] + ) @pytest.mark.asyncio @@ -2253,7 +2509,7 @@ async def test_create_table_integer_default_expr( ds_write, default_expr, minimum_value, expected_schema ): token = write_token(ds_write) - table = "default_{}".format(default_expr) + table = f"default_{default_expr}" response = await ds_write.client.post( "/data/-/create", json={ @@ -2281,7 +2537,7 @@ async def test_create_table_integer_default_expr( row = await db.execute_write_fn( lambda conn: _insert_and_fetch_created( - conn, table, "insert into {} default values".format(escape_sqlite(table)) + conn, table, f"insert into {escape_sqlite(table)} default values" ) ) assert row[0] > minimum_value @@ -2331,13 +2587,12 @@ async def test_create_table_column_validation(ds_write, column, expected_error): ) if expected_error: assert response.status_code == 400 - assert response.json() == {"ok": False, "errors": [expected_error]} + assert response.json() == error_body([expected_error], 400) else: assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["Could not detect single primary key for table 'owners'"], - } + assert response.json() == error_body( + ["Could not detect single primary key for table 'owners'"], 400 + ) @pytest.mark.asyncio @@ -2375,10 +2630,9 @@ async def test_create_table_foreign_key_without_fk_column_requires_single_pk(ds_ headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["Could not detect single primary key for table 'accounts'"], - } + assert response.json() == error_body( + ["Could not detect single primary key for table 'accounts'"], 400 + ) @pytest.mark.asyncio @@ -2528,10 +2782,9 @@ async def test_create_table_error_if_pk_changed(ds_write): headers=_headers(token), ) assert second_response.status_code == 400 - assert second_response.json() == { - "ok": False, - "errors": ["pk cannot be changed for existing table"], - } + assert second_response.json() == error_body( + ["pk cannot be changed for existing table"], 400 + ) @pytest.mark.asyncio @@ -2555,10 +2808,9 @@ async def test_create_table_error_rows_twice_with_duplicates(ds_write): headers=_headers(token), ) assert second_response.status_code == 400 - assert second_response.json() == { - "ok": False, - "errors": ["UNIQUE constraint failed: test_create_twice.id"], - } + assert second_response.json() == error_body( + ["UNIQUE constraint failed: test_create_twice.id"], 400 + ) @pytest.mark.asyncio @@ -2581,6 +2833,8 @@ async def test_method_not_allowed(ds_write, path): assert response.json() == { "ok": False, "error": "Method not allowed", + "errors": ["Method not allowed"], + "status": 405, } @@ -2648,10 +2902,9 @@ async def test_create_using_alter_against_existing_table( ) if not has_alter_permission: assert response2.status_code == 403 - assert response2.json() == { - "ok": False, - "errors": ["Permission denied: need alter-table"], - } + assert response2.json() == error_body( + ["Permission denied: need alter-table"], 403 + ) else: assert response2.status_code == 201 @@ -2667,3 +2920,119 @@ async def test_create_using_alter_against_existing_table( insert_rows_event = ds_write._tracked_events[1] assert insert_rows_event.name == "insert-rows" assert insert_rows_event.num_rows == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("denied_action", "request_body"), + ( + ( + "insert-row", + { + "table": "salaries", + "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}], + }, + ), + ( + "update-row", + { + "table": "salaries", + "rows": [{"id": 1, "note": "REPLACED"}], + "pk": "id", + "replace": True, + }, + ), + ( + "alter-table", + { + "table": "salaries", + "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}], + "alter": True, + }, + ), + ), +) +async def test_create_table_existing_table_respects_table_level_denial( + denied_action, request_body +): + # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table + # inserts rows into it, so insert-row (and update-row / alter-table) must be + # checked against the TableResource, not just the DatabaseResource. + ds = Datasette( + memory=True, + config={ + "databases": { + # id=editor user has each permission at the database level, but + # the selected action is explicitly denied on the salaries table + "data": { + "permissions": { + "create-table": {"id": "editor"}, + "insert-row": {"id": "editor"}, + "update-row": {"id": "editor"}, + "alter-table": {"id": "editor"}, + }, + "tables": { + "salaries": {"permissions": {denied_action: False}}, + }, + } + } + }, + ) + db = ds.add_memory_database( + f"create_table_existing_table_denied_{denied_action}", name="data" + ) + await db.execute_write("create table salaries (id integer primary key, note text)") + await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')") + await ds.invoke_startup() + + if denied_action == "insert-row": + # Sanity: direct insert into salaries is denied for this actor + direct = await ds.client.post( + "/data/salaries/-/insert", + actor={"id": "editor"}, + json={"row": {"id": 9, "note": "INJ-DIRECT"}}, + ) + assert direct.status_code == 403 + + response = await ds.client.post( + "/data/-/create", + actor={"id": "editor"}, + json=request_body, + ) + assert response.status_code == 403, response.json() + assert response.json()["errors"] == [f"Permission denied: need {denied_action}"] + rows = (await db.execute("select id, note from salaries order by id")).rows + assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")] + assert await db.table_columns("salaries") == ["id", "note"] + + +@pytest.mark.asyncio +async def test_create_table_respects_predeclared_table_level_denial(): + ds = Datasette( + memory=True, + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "editor"}, + "insert-row": {"id": "editor"}, + }, + "tables": { + "planned_table": {"permissions": {"insert-row": False}}, + }, + } + } + }, + ) + db = ds.add_memory_database("create_table_predeclared_denial", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/create", + actor={"id": "editor"}, + json={"table": "planned_table", "rows": [{"id": 1}]}, + ) + + assert response.status_code == 403, response.json() + assert response.json()["errors"] == ["Permission denied: need insert-row"] + assert not await db.table_exists("planned_table") diff --git a/tests/test_auth.py b/tests/test_auth.py index 5868a21c..68a2e6fd 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,14 +1,18 @@ +import time +from unittest.mock import AsyncMock + +import pytest from bs4 import BeautifulSoup as Soup -from .utils import cookie_was_deleted, last_event from click.testing import CliRunner -from datasette.utils import baseconv + from datasette.cli import cli from datasette.resources import ( DatabaseResource, TableResource, ) -import pytest -import time +from datasette.utils import baseconv + +from .utils import cookie_was_deleted, last_event @pytest.mark.asyncio @@ -204,7 +208,7 @@ def test_auth_create_token( assert response2.status == 200 if errors: for error in errors: - assert '

      {}

      '.format(error) in response2.text + assert f'

      {error}

      ' in response2.text else: # Check create-token event event = last_event(app_client.ds) @@ -228,18 +232,49 @@ def test_auth_create_token( # And test that token response3 = app_client.get( "/-/actor.json", - headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, + headers={"Authorization": "Bearer {}".format(f"dstok_{token}")}, ) assert response3.status == 200 assert response3.json["actor"]["id"] == "test" +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["GET", "POST"]) +@pytest.mark.parametrize( + "restrictions", + [ + {}, + {"a": ["vi"]}, + {"d": {"db": ["vd"]}}, + {"r": {"db": {"t1": ["vt"]}}}, + ], + ids=["empty", "instance", "database", "table"], +) +async def test_auth_create_token_not_allowed_for_restricted_actors( + bare_ds, monkeypatch, method, restrictions +): + create_token = AsyncMock() + monkeypatch.setattr(bare_ds, "create_token", create_token) + + response = await bare_ds.client.request( + method, + "/-/create-token", + actor={"id": "test", "_r": restrictions}, + ) + + assert response.status_code == 403 + assert "Restricted actors cannot create API tokens" in response.text + create_token.assert_not_called() + + @pytest.mark.asyncio async def test_auth_create_token_not_allowed_for_tokens(ds_client): - ds_tok = ds_client.ds.sign({"a": "test", "token": "dstok"}, "token") + ds_tok = ds_client.ds.sign( + {"a": "test", "token": "dstok", "t": int(time.time())}, "token" + ) response = await ds_client.get( "/-/create-token", - headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, + headers={"Authorization": f"Bearer dstok_{ds_tok}"}, ) assert response.status_code == 403 @@ -284,17 +319,17 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): elif scenario == "invalid_token": token = "invalid" if token: - token = "dstok_{}".format(token) + token = f"dstok_{token}" if scenario == "allow_signed_tokens_off": ds_client.ds._settings["allow_signed_tokens"] = False headers = {} if token: - headers["Authorization"] = "Bearer {}".format(token) + headers["Authorization"] = f"Bearer {token}" response = await ds_client.get("/-/actor.json", headers=headers) try: if should_work: data = response.json() - assert data.keys() == {"actor"} + assert data.keys() == {"ok", "actor"} actor = data["actor"] expected_keys = {"id", "token"} if scenario != "valid_unlimited_token": @@ -304,8 +339,16 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): assert actor["token"] == "dstok" if scenario != "valid_unlimited_token": assert isinstance(actor["token_expires"], int) + elif scenario == "no_token": + # No credentials presented - request proceeds as anonymous + assert response.json() == {"ok": True, "actor": None} else: - assert response.json() == {"actor": None} + # Invalid credentials presented - hard 401 + assert response.status_code == 401 + data = response.json() + assert data["ok"] is False + assert data["status"] == 401 + assert response.headers["www-authenticate"].startswith("Bearer") finally: ds_client.ds._settings["allow_signed_tokens"] = True @@ -328,7 +371,7 @@ def test_cli_create_token(app_client, expires): assert details.keys() == expected_keys assert details["a"] == "test" response = app_client.get( - "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} ) if expires is None or expires > 0: expected_actor = { @@ -337,10 +380,11 @@ def test_cli_create_token(app_client, expires): } if expires and expires > 0: expected_actor["token_expires"] = details["t"] + expires - assert response.json == {"actor": expected_actor} + assert response.json == {"ok": True, "actor": expected_actor} else: - expected_actor = None - assert response.json == {"actor": expected_actor} + # Expired token - hard 401 + assert response.status == 401 + assert response.json["ok"] is False @pytest.mark.asyncio @@ -510,3 +554,25 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): ) is not True ), "Root without root_enabled should not automatically get set-column-type" + + +@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60)) +def test_set_actor_cookie_honours_expire_after(expire_after): + # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of + # seconds, but every value was being replaced with 24 hours. + from datasette.app import Datasette + from datasette.utils.asgi import Response + + ds = Datasette(memory=True) + response = Response.text("") + before = int(time.time()) + ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after) + after = int(time.time()) + + (header,) = response._set_cookie_headers + assert header.startswith("ds_actor=") + value = header[len("ds_actor=") :].split(";", 1)[0] + data = ds.unsign(value, "actor") + assert data["a"] == {"id": "test"} + expires_at = baseconv.base62.decode(data["e"]) + assert before + expire_after <= expires_at <= after + expire_after diff --git a/tests/test_autocomplete.py b/tests/test_autocomplete.py index 76b9c902..194fcf01 100644 --- a/tests/test_autocomplete.py +++ b/tests/test_autocomplete.py @@ -25,13 +25,14 @@ async def test_autocomplete_single_pk_exact_match_and_label_order(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 2}, "label": "Longer non-label pk match"}, {"pks": {"id": 20}, "label": "2"}, {"pks": {"id": 21}, "label": "22"}, {"pks": {"id": 3}, "label": "A label containing 2"}, {"pks": {"id": 200}, "label": "A"}, - ] + ], } @@ -52,12 +53,12 @@ async def test_autocomplete_blank_q_returns_no_results(): response = await ds.client.get("/autocomplete_blank/people/-/autocomplete?q=") assert response.status_code == 200 - assert response.json() == {"rows": []} + assert response.json() == {"ok": True, "rows": []} response = await ds.client.get("/autocomplete_blank/people/-/autocomplete") assert response.status_code == 200 - assert response.json() == {"rows": []} + assert response.json() == {"ok": True, "rows": []} @pytest.mark.asyncio @@ -81,11 +82,12 @@ async def test_autocomplete_initial_returns_latest_rows(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 3}, "label": "Cleo"}, {"pks": {"id": 2}, "label": "Bob"}, {"pks": {"id": 1}, "label": "Alice"}, - ] + ], } response = await ds.client.get( @@ -94,11 +96,12 @@ async def test_autocomplete_initial_returns_latest_rows(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 3}, "label": "Cleo"}, {"pks": {"id": 2}, "label": "Bob"}, {"pks": {"id": 1}, "label": "Alice"}, - ] + ], } @@ -121,9 +124,10 @@ async def test_autocomplete_escapes_like_characters(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 1}, "label": "100% real"}, - ] + ], } @@ -149,11 +153,12 @@ async def test_autocomplete_compound_pk_searches_all_pk_columns(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"country": "mx", "code": "ca"}, "label": "Campeche"}, {"pks": {"country": "us", "code": "ca"}, "label": "California"}, {"pks": {"country": "ca", "code": "bc"}, "label": "British Columbia"}, - ] + ], } @@ -184,9 +189,10 @@ async def test_autocomplete_primary_key_called_label(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"label": "abc"}, "label": "Display value"}, - ] + ], } @@ -246,8 +252,9 @@ async def test_autocomplete_timeout_uses_prefix_fallback(monkeypatch): assert timeout_was_simulated data = response.json() assert data == { + "ok": True, "rows": [ {"pks": {"id": f"item-1999{i:02d}"}, "label": f"name 1999{i:02d}"} for i in range(10) - ] + ], } diff --git a/tests/test_background_tasks.py b/tests/test_background_tasks.py new file mode 100644 index 00000000..c0d7ce49 --- /dev/null +++ b/tests/test_background_tasks.py @@ -0,0 +1,381 @@ +""" +Tests for datasette.add_background_task() / start_background_tasks() and the +BackgroundTask / BackgroundTaskSupervisor machinery in +datasette/background_tasks.py. +""" + +import asyncio +import contextlib +import logging + +import httpx2 +import pytest + +from datasette import hookimpl +from datasette.app import Datasette +from datasette.plugins import pm + + +async def _drive_lifespan_startup(app): + """Send a single lifespan.startup message into app's ASGI lifespan loop + and return the list of messages sent back, without ever sending + lifespan.shutdown. Copied from tests/test_lifespan.py's helper of the + same name - mirrors what a real server does: after startup completes + it parks waiting for the next event, and we cancel that wait once + we've observed the startup response. + """ + messages_sent = [] + startup_responded = asyncio.Event() + delivered = False + + async def receive(): + nonlocal delivered + if not delivered: + delivered = True + return {"type": "lifespan.startup"} + await asyncio.Event().wait() + + async def send(message): + messages_sent.append(message) + startup_responded.set() + + task = asyncio.create_task(app({"type": "lifespan"}, receive, send)) + try: + await asyncio.wait_for(startup_responded.wait(), timeout=5) + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + return messages_sent + + +@pytest.mark.asyncio +async def test_tasks_registered_in_startup_hook_run_after_lifespan_startup(): + # Two tasks registered by one plugin's startup hook - order preserved, + # both running after lifespan startup completes, and no HTTP request + # of any kind is issued anywhere in this test. + events = [] + + async def task_one(datasette): + events.append("task_one") + # Wait indefinitely to simulate long-lived background work, keeping + # the task "running" for the assertions below until cleanup cancels it. + await asyncio.Event().wait() + + async def task_two(datasette): + events.append("task_two") + await asyncio.Event().wait() + + class TwoTaskPlugin: + __name__ = "TwoTaskPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + datasette.add_background_task(task_one, name="task-one") + datasette.add_background_task(task_two, name="task-two") + + return inner + + ds = Datasette(memory=True) + pm.register(TwoTaskPlugin(), name="two_task_plugin") + try: + app = ds.app() + messages = await _drive_lifespan_startup(app) + assert {"type": "lifespan.startup.complete"} in messages + + handles = ds._background_tasks.tasks() + assert [h.name for h in handles] == ["task-one", "task-two"] + + # Let both tasks run their first line of code. + await asyncio.sleep(0) + assert handles[0].state == "running" + assert handles[1].state == "running" + assert events == ["task_one", "task_two"] + finally: + pm.unregister(name="two_task_plugin") + await ds._background_tasks.cancel_all(grace=1.0) + + +@pytest.mark.asyncio +async def test_launch_waits_for_every_startup_hook_before_running_any_task(): + # PluginA registers a task from its startup hook; PluginB does the + # same from ITS startup hook, which runs after PluginA's (forced with + # tryfirst=True on A). Even though A's registration happens first, + # A's task body must not actually execute until every startup hook - + # including B's - has finished, since launch only happens after + # invoke_startup() completes. This is the ordering guarantee that + # dissolves datasette-cron's tryfirst=True launch hack. + hook_call_order = [] + seen_names_when_a_ran = {} + + async def task_a(datasette): + seen_names_when_a_ran["names"] = [ + h.name for h in datasette._background_tasks.tasks() + ] + + async def task_b(datasette): + pass + + class PluginA: + __name__ = "PluginA" + + @hookimpl(tryfirst=True) + def startup(self, datasette): + async def inner(): + hook_call_order.append("A") + datasette.add_background_task(task_a, name="task-a") + + return inner + + class PluginB: + __name__ = "PluginB" + + @hookimpl + def startup(self, datasette): + async def inner(): + hook_call_order.append("B") + datasette.add_background_task(task_b, name="task-b") + + return inner + + ds = Datasette(memory=True) + pm.register(PluginA(), name="plugin_a") + pm.register(PluginB(), name="plugin_b") + try: + await ds.start_background_tasks() + # Confirm A's startup hook really did run (and register task-a) + # strictly before B's startup hook ran. + assert hook_call_order == ["A", "B"] + + handles = ds._background_tasks.tasks() + await asyncio.wait_for(asyncio.gather(*[h.task for h in handles]), timeout=5) + # Yet by the time task-a's own body executed (after launch, which + # only happens once every startup hook - including B's - has + # finished), task-b was already registered. + assert "task-b" in seen_names_when_a_ran["names"] + finally: + pm.unregister(name="plugin_a") + pm.unregister(name="plugin_b") + + +@pytest.mark.asyncio +async def test_concurrent_first_requests_launch_background_tasks_exactly_once(): + launch_count = {"n": 0} + + async def counting_task(datasette): + launch_count["n"] += 1 + + class CountingTaskPlugin: + __name__ = "CountingTaskPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + datasette.add_background_task(counting_task, name="counting-task") + + return inner + + ds = Datasette(memory=True) + pm.register(CountingTaskPlugin(), name="counting_task_plugin") + try: + app = ds.app() + transport = httpx2.ASGITransport(app=app) + async with httpx2.AsyncClient( + transport=transport, base_url="http://localhost" + ) as client: + responses = await asyncio.gather( + *[client.get("/-/versions.json") for _ in range(10)] + ) + assert all(response.status_code == 200 for response in responses) + + handles = ds._background_tasks.tasks() + assert len(handles) == 1 + await asyncio.wait_for(handles[0].task, timeout=5) + assert launch_count["n"] == 1 + finally: + pm.unregister(name="counting_task_plugin") + + +@pytest.mark.asyncio +async def test_post_launch_registration_starts_immediately_and_cancel_works(): + ds = Datasette(memory=True) + await ds.start_background_tasks() # nothing registered yet, but launched + + started = asyncio.Event() + + async def long_running(datasette): + started.set() + await asyncio.Event().wait() + + handle = ds.add_background_task(long_running, name="dynamic-task") + # Registered after launch: starts immediately rather than sitting in + # "registered" limbo. + assert handle.state == "running" + assert handle.task is not None + + await asyncio.wait_for(started.wait(), timeout=5) + assert handle.state == "running" + + handle.cancel() + with pytest.raises(asyncio.CancelledError): + await handle.task + await asyncio.sleep(0) + assert handle.state == "cancelled" + + +@pytest.mark.asyncio +async def test_pre_launch_registration_starts_as_registered(): + ds = Datasette(memory=True) + + async def task(datasette): + pass + + handle = ds.add_background_task(task, name="buffered-task") + assert handle.state == "registered" + assert handle.task is None + + handle.cancel() # not yet launched: deregisters instead of cancelling + assert handle not in ds._background_tasks.tasks() + + +@pytest.mark.asyncio +async def test_crashing_task_logs_traceback_and_state_is_crashed(caplog): + ds = Datasette(memory=True) + await ds.start_background_tasks() + + survivor_ran = asyncio.Event() + + async def crashing_task(datasette): + raise RuntimeError("kaboom") + + async def survivor(datasette): + survivor_ran.set() + + with caplog.at_level(logging.ERROR, logger="datasette.background_tasks"): + crash_handle = ds.add_background_task(crashing_task, name="crashing_task") + survivor_handle = ds.add_background_task(survivor, name="survivor") + await asyncio.wait_for( + asyncio.gather( + crash_handle.task, survivor_handle.task, return_exceptions=True + ), + timeout=5, + ) + + assert crash_handle.state == "crashed" + assert isinstance(crash_handle.exception, RuntimeError) + assert str(crash_handle.exception) == "kaboom" + + # The crash must not affect any other task. + assert survivor_ran.is_set() + assert survivor_handle.state == "completed" + + assert "crashing_task" in caplog.text + assert "kaboom" in caplog.text + assert "Traceback" in caplog.text + assert "RuntimeError" in caplog.text + + +def test_name_collisions_get_suffixed_and_explicit_names_are_respected(): + ds = Datasette(memory=True) + + async def noop(datasette): + pass + + async def another_noop(datasette): + pass + + h1 = ds.add_background_task(noop, name="dup") + h2 = ds.add_background_task(another_noop, name="dup") + h3 = ds.add_background_task(noop, name="dup") + assert [h1.name, h2.name, h3.name] == ["dup", "dup-2", "dup-3"] + + h_explicit = ds.add_background_task(noop, name="explicit-name") + assert h_explicit.name == "explicit-name" + + h_default = ds.add_background_task(noop) + assert h_default.name == noop.__qualname__ + + +@pytest.mark.asyncio +async def test_start_background_tasks_on_bare_datasette(): + # The headless-CLI path (datasette-rss's `fetch --due` shape): no + # server, no lifespan, no first HTTP request - just an explicit call. + ran = asyncio.Event() + + async def task(datasette): + ran.set() + + ds = Datasette([]) + assert ds._startup_invoked is False + + handle = ds.add_background_task(task, name="headless-task") + assert handle.state == "registered" + + await ds.start_background_tasks() + + assert ds._startup_invoked is True + await asyncio.wait_for(ran.wait(), timeout=5) + await asyncio.wait_for(handle.task, timeout=5) + # handle.task being done only guarantees the coroutine has returned, + # not that our done-callback (which updates handle.state) has run yet - + # asyncio schedules done-callbacks via call_soon, and awaiting an + # already-done future/task returns immediately without giving the loop + # a chance to drain its ready queue. Yield once to let it run. + await asyncio.sleep(0) + assert handle.state == "completed" + + +@pytest.mark.asyncio +async def test_cancel_all_cancels_running_tasks_and_leaves_completed_alone(): + ds = Datasette(memory=True) + await ds.start_background_tasks() + + async def forever(datasette): + await asyncio.Event().wait() + + async def quick(datasette): + return "done" + + forever_handle = ds.add_background_task(forever, name="forever") + quick_handle = ds.add_background_task(quick, name="quick") + await asyncio.wait_for(quick_handle.task, timeout=5) + assert quick_handle.state == "completed" + + await ds._background_tasks.cancel_all(grace=1.0) + + assert forever_handle.state == "cancelled" + assert quick_handle.state == "completed" + + +@pytest.mark.asyncio +async def test_cancel_all_logs_stragglers_that_outlive_the_grace_period(caplog): + ds = Datasette(memory=True) + await ds.start_background_tasks() + + async def stubborn(datasette): + with contextlib.suppress(asyncio.CancelledError): + await asyncio.sleep(10) + # Swallowing CancelledError above and returning normally simulates + # a task that ignores cancellation for longer than the grace period. + await asyncio.sleep(10) + + handle = ds.add_background_task(stubborn, name="stubborn-task") + # Let the task actually start running and reach its first sleep (inside + # the CancelledError-suppressing block) before cancelling it - a task + # cancelled before it has ever run its first step never enters that + # block at all (the throw happens before the coroutine body starts), + # so it would finish cancelling immediately instead of behaving like a + # straggler. + await asyncio.sleep(0) + + with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"): + await ds._background_tasks.cancel_all(grace=0.1) + + assert "stubborn-task" in caplog.text + + # Clean up: actually cancel it now that the test has made its + # assertion, so it doesn't leak past the end of the test. + handle.task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await handle.task diff --git a/tests/test_base_view.py b/tests/test_base_view.py index 2cd4d601..b46f7ce1 100644 --- a/tests/test_base_view.py +++ b/tests/test_base_view.py @@ -1,8 +1,10 @@ -from datasette.views.base import View +import json + +import pytest + from datasette import Request, Response from datasette.app import Datasette -import json -import pytest +from datasette.views.base import View class GetView(View): @@ -53,6 +55,8 @@ async def test_get_view(): assert json.loads(post_json_response.body) == { "ok": False, "error": "Method not allowed", + "errors": ["Method not allowed"], + "status": 405, } assert post_json_response.status == 405 diff --git a/tests/test_cli.py b/tests/test_cli.py index f86d6909..fbd4a8a9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,23 +1,28 @@ -from .fixtures import ( - make_app_client, - TestClient as _TestClient, - EXPECTED_PLUGINS, -) -from datasette.app import SETTINGS -from datasette.plugins import DEFAULT_PLUGINS, pm -from datasette.cli import cli, serve -from datasette.version import __version__ -from datasette.utils import tilde_encode -from datasette.utils.sqlite import sqlite3 -from click.testing import CliRunner import io import json import pathlib -import pytest import sys import textwrap from unittest import mock +import pytest +from click.testing import CliRunner + +from datasette.app import SETTINGS +from datasette.cli import cli, serve +from datasette.plugins import DEFAULT_PLUGINS, pm +from datasette.utils import tilde_encode +from datasette.utils.sqlite import sqlite3 +from datasette.version import __version__ + +from .fixtures import ( + EXPECTED_PLUGINS, + make_app_client, +) +from .fixtures import ( + TestClient as _TestClient, +) + def test_inspect_cli(app_client): runner = CliRunner() @@ -385,7 +390,9 @@ def test_setting_boolean_validation_false_values(value): ) # Should be forbidden (setting is false) assert result.exit_code == 1, result.output - assert "Forbidden" in result.output + error = json.loads(result.output) + assert error["ok"] is False + assert error["status"] == 403 @pytest.mark.parametrize("value", ("on", "true", "1")) @@ -425,8 +432,9 @@ def test_setting_default_allow_sql(default_allow_sql): assert json.loads(result.output)["rows"][0] == {"21": 21} else: assert result.exit_code == 1, result.output - # This isn't JSON at the moment, maybe it should be though - assert "Forbidden" in result.output + error = json.loads(result.output) + assert error["ok"] is False + assert error["status"] == 403 def test_sql_errors_logged_to_stderr(): @@ -444,7 +452,7 @@ def test_serve_create(tmpdir): cli, [str(db_path), "--create", "--get", "/-/databases.json"] ) assert result.exit_code == 0, result.output - databases = json.loads(result.output) + databases = json.loads(result.output)["databases"] assert { "name": "does_not_exist_yet", "is_mutable": True, @@ -457,7 +465,7 @@ def test_serve_create(tmpdir): @pytest.mark.parametrize("argument", ("-c", "--config")) @pytest.mark.parametrize("format_", ("json", "yaml")) def test_serve_config(tmpdir, argument, format_): - config_path = tmpdir / "datasette.{}".format(format_) + config_path = tmpdir / f"datasette.{format_}" config_path.write_text( ( "settings:\n default_page_size: 5\n" @@ -493,7 +501,7 @@ def test_serve_duplicate_database_names(tmpdir): conn.close() result = runner.invoke(cli, [db_1_path, db_2_path, "--get", "/-/databases.json"]) assert result.exit_code == 0, result.output - databases = json.loads(result.output) + databases = json.loads(result.output)["databases"] assert {db["name"] for db in databases} == {"db", "db_2"} @@ -510,13 +518,13 @@ def test_weird_database_names(tmpdir, filename): result1 = runner.invoke(cli, [db_path, "--get", "/"]) assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] - expected_link = '{}'.format( - tilde_encode(filename_no_stem), filename_no_stem + expected_link = ( + f'{filename_no_stem}' ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))] + cli, [db_path, "--get", f"/{tilde_encode(filename_no_stem)}"] ) assert result2.exit_code == 0, result2.output @@ -585,7 +593,7 @@ def test_duplicate_database_files_error(tmpdir): cli, ["serve", other_db_path, str(config_dir), "--get", "/-/databases.json"] ) assert result4.exit_code == 0 - databases = json.loads(result4.output) + databases = json.loads(result4.output)["databases"] assert {db["name"] for db in databases} == {"other", "data"} # Test that multiple directories raise an error diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index dc852201..562bd7aa 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,8 +1,10 @@ +import json +import textwrap + +from click.testing import CliRunner + from datasette.cli import cli from datasette.plugins import pm -from click.testing import CliRunner -import textwrap -import json def test_serve_with_get(tmp_path_factory): @@ -44,9 +46,59 @@ def test_serve_with_get(tmp_path_factory): # Annoyingly that new test plugin stays resident - we need # to manually unregister it to avoid conflict with other tests - to_unregister = [ + to_unregister = next( p for p in pm.get_plugins() if p.__name__ == "init_for_serve_with_get.py" - ][0] + ) + pm.unregister(to_unregister) + + +def test_serve_with_get_does_not_launch_background_tasks(tmp_path_factory): + # --get must never launch background tasks, even though its TestClient + # request + # flows through the full ASGI stack (including the AsgiRunOnFirstRequest + # fallback that would otherwise launch them). The plugin's startup hook + # itself still runs (registration happens) - only the launch is + # suppressed, so the sentinel file the background task would write must + # never appear. + plugins_dir = tmp_path_factory.mktemp("plugins_for_get_background_tasks") + sentinel = plugins_dir / "sentinel.txt" + (plugins_dir / "bg_task_for_get.py").write_text( + textwrap.dedent( + f""" + from datasette import hookimpl + + @hookimpl + def startup(datasette): + async def inner(): + async def task(datasette): + with open("{sentinel!s}", "w") as fp: + fp.write("ran") + + datasette.add_background_task(task, name="get-sentinel-task") + + return inner + """, + ), + "utf-8", + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "serve", + "--memory", + "--plugins-dir", + str(plugins_dir), + "--get", + "/_memory/-/query.json?sql=select+1", + ], + ) + assert result.exit_code == 0, result.output + assert not sentinel.exists() + + to_unregister = next( + p for p in pm.get_plugins() if p.__name__ == "bg_task_for_get.py" + ) pm.unregister(to_unregister) @@ -95,7 +147,10 @@ def test_serve_with_get_and_token(): ], ) assert 0 == result2.exit_code, result2.output - assert json.loads(result2.output) == {"actor": {"id": "root", "token": "dstok"}} + assert json.loads(result2.output) == { + "ok": True, + "actor": {"id": "root", "token": "dstok"}, + } def test_serve_with_get_exit_code_for_error(): @@ -130,8 +185,9 @@ def test_serve_get_actor(): ) assert result.exit_code == 0 assert json.loads(result.output) == { + "ok": True, "actor": { "id": "root", "extra": "x", - } + }, } diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 47f23c08..163d3fcc 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,11 +1,15 @@ -import httpx -import pytest +import signal import socket +import subprocess +import time + +import httpx2 +import pytest @pytest.mark.serial def test_serve_localhost_http(ds_localhost_http_server): - response = httpx.get("http://localhost:8041/_memory.json") + response = httpx2.get("http://localhost:8041/_memory.json") assert { "database": "_memory", "path": "/_memory", @@ -19,11 +23,205 @@ def test_serve_localhost_http(ds_localhost_http_server): ) def test_serve_unix_domain_socket(ds_unix_domain_socket_server): _, uds = ds_unix_domain_socket_server - transport = httpx.HTTPTransport(uds=uds) - client = httpx.Client(transport=transport) - response = client.get("http://localhost/_memory.json") + transport = httpx2.HTTPTransport(uds=uds) + with httpx2.Client(transport=transport) as client: + response = client.get("http://localhost/_memory.json") assert { "database": "_memory", "path": "/_memory", "tables": [], }.items() <= response.json().items() + + +# Shaped after datasette-litestream's startup hook, which schedules a +# background task with asyncio.get_running_loop().create_task(...): +# https://github.com/datasette/datasette-litestream +MARKER_TASK_PLUGIN = """ +import asyncio +from datasette import hookimpl +from datasette.utils.asgi import Response + + +@hookimpl +def startup(datasette): + datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1 + + async def _mark(): + # Must await before setting the flag: a task with no internal + # await point could finish on the throwaway loop before it + # closed, masking the regression this test guards against. + await asyncio.sleep(0.2) + datasette._marker_task_ran = True + + asyncio.get_running_loop().create_task(_mark()) + + +@hookimpl +def register_routes(): + async def marker_status(datasette): + return Response.json( + { + "marker_task_ran": getattr(datasette, "_marker_task_ran", False), + "startup_calls": getattr(datasette, "_startup_calls", 0), + } + ) + + return [(r"^/-/marker-task-ran$", marker_status)] +""" + + +STARTUP_ERROR_PLUGIN = """ +from datasette import hookimpl +from datasette.utils import StartupError + + +@hookimpl +def startup(datasette): + raise StartupError("boom from plugin") +""" + + +@pytest.mark.serial +def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins): + """ + Litestream-shaped regression test: a startup hook that does + asyncio.get_running_loop().create_task(...) must have that task + actually execute before/while the server is handling requests. This + only holds if invoke_startup() and uvicorn.Server.serve() share one + event loop. This test fails against unmodified main, where + invoke_startup() runs on a throwaway loop that is closed before + uvicorn opens its own loop to serve. + """ + _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN}) + # The fixture has already waited for the server to answer requests. The + # marker task deliberately awaits before setting its flag, so poll for a + # moment rather than assuming it landed before the first request arrived. + deadline = time.time() + 3.0 + payload = {} + while time.time() < deadline: + payload = httpx2.get( + f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0 + ).json() + if payload["marker_task_ran"]: + break + time.sleep(0.05) + assert payload.get("marker_task_ran"), ( + "The startup hook's asyncio.create_task(...) never ran - " + "invoke_startup() and the server are not sharing an event loop" + ) + # Polling above means this test would also pass if the startup hook were + # re-run on the serving loop by the first-request fallback - which would + # hide exactly the bug being tested. invoke_startup() is idempotent today + # so that cannot happen; assert it explicitly so that if the idempotency + # guard is ever removed this test fails loudly instead of silently + # becoming a no-op. + assert payload["startup_calls"] == 1, ( + "startup hook ran {} times - the marker may have been set by a " + "re-run on the serving loop rather than by the original task".format( + payload["startup_calls"] + ) + ) + + +@pytest.mark.serial +def test_startup_error_fails_fast_before_port_binds(serve_with_plugins): + """ + A "startup" plugin hook that raises StartupError must fail fast: print + the message, exit non-zero, and never accept a connection on the port - + the failure must happen before uvicorn.Server binds the socket. + """ + proc, port = serve_with_plugins( + {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False + ) + stdout, _ = proc.communicate(timeout=15) + output = stdout.decode("utf-8") + assert proc.returncode not in (0, None), output + assert "boom from plugin" in output, output + + # Nothing is listening on the port now the process has exited. This + # confirms the socket was not left bound; on its own it cannot prove the + # failure preceded the bind, since a port nothing ever touched also + # refuses connections. + with ( + pytest.raises(OSError), + socket.create_connection(("127.0.0.1", port), timeout=0.2), + ): + pass + + +# Verify that SIGTERM and SIGINT sent to `datasette serve` trigger uvicorn's +# lifespan.shutdown event and run the plugin shutdown hooks. The plugin below +# writes a sentinel file from its shutdown hook so the tests can check that +# cleanup ran after the server subprocess exits. +SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE = """ +import pathlib +from datasette import hookimpl + +SENTINEL_PATH = {sentinel_path!r} + + +@hookimpl +def shutdown(datasette): + pathlib.Path(SENTINEL_PATH).write_text("shutdown ran", "utf-8") +""" + + +def _start_serve_with_shutdown_sentinel(serve_with_plugins, tmp_path): + sentinel_path = tmp_path / "shutdown-sentinel.txt" + proc, _ = serve_with_plugins( + { + "shutdown_sentinel_plugin": SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE.format( + sentinel_path=str(sentinel_path) + ) + } + ) + return proc, sentinel_path + + +@pytest.mark.serial +def test_sigterm_runs_shutdown_hooks(serve_with_plugins, tmp_path): + ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel( + serve_with_plugins, tmp_path + ) + assert not sentinel_path.exists() + ds_proc.send_signal(signal.SIGTERM) + try: + ds_proc.wait(timeout=10) + except subprocess.TimeoutExpired: + ds_proc.kill() + ds_proc.wait() + raise AssertionError( + "datasette serve did not exit within 10s of SIGTERM\n" + + ds_proc.stdout.read().decode("utf-8") + ) + output = ds_proc.stdout.read().decode("utf-8") + assert sentinel_path.exists(), ( + "shutdown hook never wrote its sentinel file after SIGTERM\n" + output + ) + assert sentinel_path.read_text("utf-8") == "shutdown ran" + + +@pytest.mark.serial +@pytest.mark.skipif( + not hasattr(signal, "SIGINT"), reason="Requires signal.SIGINT support" +) +def test_sigint_runs_shutdown_hooks(serve_with_plugins, tmp_path): + ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel( + serve_with_plugins, tmp_path + ) + assert not sentinel_path.exists() + ds_proc.send_signal(signal.SIGINT) + try: + ds_proc.wait(timeout=10) + except subprocess.TimeoutExpired: + ds_proc.kill() + ds_proc.wait() + raise AssertionError( + "datasette serve did not exit within 10s of SIGINT\n" + + ds_proc.stdout.read().decode("utf-8") + ) + output = ds_proc.stdout.read().decode("utf-8") + assert sentinel_path.exists(), ( + "shutdown hook never wrote its sentinel file after SIGINT\n" + output + ) + assert sentinel_path.read_text("utf-8") == "shutdown ran" diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 45a9e7d1..d8dfc627 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -1,7 +1,11 @@ import json import logging +import time +import markupsafe +import pytest from bs4 import BeautifulSoup as Soup + from datasette.app import Datasette from datasette.column_types import ( ColumnType, @@ -9,11 +13,7 @@ from datasette.column_types import ( ) from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.utils import sqlite3 -from datasette.utils import StartupError -import markupsafe -import pytest -import time +from datasette.utils import StartupError, error_body, sqlite3 @pytest.fixture @@ -31,6 +31,7 @@ def ds_ct(tmp_path_factory): "'https://example.com', '{\"key\": \"value\"}')" ) db.commit() + db.close() ds = Datasette( [db_path], config={ @@ -70,6 +71,7 @@ def ds_ct_editor_permission(tmp_path_factory): "'https://example.com', '{\"key\": \"value\"}')" ) db.commit() + db.close() ds = Datasette( [db_path], config={ @@ -104,7 +106,7 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": "application/json", } @@ -322,12 +324,6 @@ async def test_clear_column_type_api(ds_ct): "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ], ), - ( - {"column": "title", "column_type": {"type": "email"}}, - "invalid_content_type", - 400, - ["Invalid content-type, must be application/json"], - ), ( [], None, @@ -413,11 +409,7 @@ async def test_set_column_type_api_errors( kwargs = { "headers": { "Authorization": f"Bearer {token}", - "Content-Type": ( - "text/plain" - if special_case == "invalid_content_type" - else "application/json" - ), + "Content-Type": "application/json", } } if special_case == "invalid_json": @@ -426,7 +418,7 @@ async def test_set_column_type_api_errors( kwargs["json"] = body response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs) assert response.status_code == expected_status - assert response.json() == {"ok": False, "errors": expected_errors} + assert response.json() == error_body(expected_errors, expected_status) @pytest.mark.asyncio diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 0a9b30d8..00540464 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,10 +1,12 @@ import json import pathlib + import pytest from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3 from datasette.utils import StartupError +from datasette.utils.sqlite import sqlite3 + from .fixtures import TestClient as _TestClient PLUGIN = """ @@ -109,9 +111,10 @@ def test_settings(config_dir_client): def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status - assert "hooray.py" in {p["name"] for p in response.json} - assert "non_py_file.txt" not in {p["name"] for p in response.json} - assert "mypy_cache" not in {p["name"] for p in response.json} + plugins = response.json + assert "hooray.py" in {p["name"] for p in plugins} + assert "non_py_file.txt" not in {p["name"] for p in plugins} + assert "mypy_cache" not in {p["name"] for p in plugins} def test_templates_and_plugin(config_dir_client): @@ -136,7 +139,7 @@ def test_static_directory_browsing_not_allowed(config_dir_client): def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status - databases = response.json + databases = response.json["databases"] assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index 11e53224..ffd0870c 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -1,7 +1,9 @@ -from datasette.cli import cli -from click.testing import CliRunner -import urllib import sqlite3 +import urllib + +from click.testing import CliRunner + +from datasette.cli import cli def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled): @@ -40,7 +42,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory): db_dir = tmp_path_factory.mktemp("dbs") dbs = [] for i in range(11): - path = str(db_dir / "db_{}.db".format(i)) + path = str(db_dir / f"db_{i}.db") conn = sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 2fcfb216..6c78f69d 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -44,7 +44,7 @@ async def _run_middleware(scope): await mw(scope, None, send) if inner_called: return ("allowed",) - start = [m for m in sent if m["type"] == "http.response.start"][0] + start = next(m for m in sent if m["type"] == "http.response.start") return ("blocked", start["status"]) diff --git a/tests/test_csv.py b/tests/test_csv.py index a2f03776..adae7e24 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,8 +1,10 @@ -from datasette.app import Datasette -from bs4 import BeautifulSoup as Soup -import pytest import urllib.parse +import pytest +from bs4 import BeautifulSoup as Soup + +from datasette.app import Datasette + EXPECTED_TABLE_CSV = """id,content 1,hello 2,world @@ -164,6 +166,66 @@ async def test_custom_sql_csv(ds_client): assert response.text == EXPECTED_CUSTOM_CSV +@pytest.mark.asyncio +@pytest.mark.parametrize("download", (False, True)) +@pytest.mark.parametrize( + "query_string,expected_error", + ( + ("sql=select+blah", "no such column: blah"), + ("sql=select+*+from+missing", "no such table: missing"), + ("sql=select+from", 'near "from": syntax error'), + ( + "sql=delete+from+simple_primary_key", + "Statement must be a SELECT", + ), + ("", "?sql= is required"), + ( + "sql=select+sleep(0.01)&_timelimit=5", + ( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ), + ), + ), +) +async def test_custom_sql_csv_errors(ds_client, query_string, expected_error, download): + if download: + query_string += "&_dl=1" + response = await ds_client.get(f"/fixtures/-/query.csv?{query_string}") + assert response.status_code == 400 + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert "content-disposition" not in response.headers + assert response.text == expected_error + + +@pytest.mark.asyncio +async def test_custom_sql_csv_error_head(ds_client): + response = await ds_client.head("/fixtures/-/query.csv?sql=select+blah") + assert response.status_code == 400 + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.content == b"" + + +@pytest.mark.asyncio +async def test_custom_sql_csv_error_cors(): + ds = Datasette(cors=True) + response = await ds.client.get("/_memory/-/query.csv?sql=select+blah") + assert response.status_code == 400 + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.headers["access-control-allow-origin"] == "*" + assert response.text == "no such column: blah" + + +@pytest.mark.asyncio +async def test_table_csv_error(ds_client): + response = await ds_client.get( + "/fixtures/simple_primary_key.csv?_where=blah&_stream=1" + ) + assert response.status_code == 400 + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == "no such column: blah" + + @pytest.mark.asyncio async def test_table_csv_download(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.csv?_dl=1") @@ -227,6 +289,20 @@ async def test_table_csv_stream(ds_client): assert len([b for b in response.content.split(b"\r\n") if b]) == 1002 +@pytest.mark.asyncio +async def test_view_csv_stream(ds_client): + # Without _stream should return header + 100 rows: + response = await ds_client.get("/fixtures/paginated_view.csv?_size=max") + assert len([b for b in response.content.split(b"\r\n") if b]) == 101 + # With _stream=1 should paginate through all pages and return header + 202 rows + response = await ds_client.get("/fixtures/paginated_view.csv?_stream=1") + lines = [b for b in response.content.split(b"\r\n") if b] + assert len(lines) == 203 + # Ensure there are no duplicate rows from looping + assert len(set(lines[1:])) == 202 + assert lines[0] == b"content,content_extra" + + def test_csv_trace(app_client_with_trace): response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1") assert response.headers["content-type"] == "text/html; charset=utf-8" diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 86cdcc6b..32cfc43d 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -1,5 +1,7 @@ import pathlib + import pytest + from .fixtures import make_app_client TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates") diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index f1e43064..f456a17f 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -1,4 +1,5 @@ import pytest + from datasette.app import Datasette from datasette.resources import DatabaseResource, TableResource diff --git a/tests/test_docs.py b/tests/test_docs.py index 13b3a549..b36b773e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -2,20 +2,22 @@ Tests to ensure certain things are documented. """ -from datasette import app, utils +import re +from pathlib import Path + +import pytest + import datasette.fixtures # noqa: F401 +from datasette import app, utils from datasette.app import Datasette from datasette.filters import Filters -from pathlib import Path -import pytest -import re docs_path = Path(__file__).parent.parent / "docs" label_re = re.compile(r"\.\. _([^\s:]+):") def get_headings(content, underline="-"): - heading_re = re.compile(r"(\w+)(\([^)]*\))?\n\{}+\n".format(underline)) + heading_re = re.compile(rf"(\w+)(\([^)]*\))?\n\{underline}+\n") return {h[0] for h in heading_re.findall(content)} @@ -25,14 +27,20 @@ def get_labels(filename): @pytest.fixture(scope="session") -def settings_headings(): - return get_headings((docs_path / "settings.rst").read_text(), "~") +def settings_sections(): + content = (docs_path / "settings.rst").read_text() + sections = re.split(r"^(\w+)\n~+\n", content, flags=re.MULTILINE) + return dict(zip(sections[1::2], sections[2::2])) -def test_settings_are_documented(settings_headings, subtests): +def test_settings_are_documented(settings_sections, subtests): for setting in app.SETTINGS: with subtests.test(setting=setting.name): - assert setting.name in settings_headings + assert setting.name in settings_sections + assert ( + f'setting_default(cog, "{setting.name}")' + in settings_sections[setting.name] + ) @pytest.fixture(scope="session") @@ -248,7 +256,7 @@ async def test_homepage(): async def test_actor_is_null(): ds = Datasette(memory=True) response = await ds.client.get("/-/actor.json") - assert response.json() == {"actor": None} + assert response.json() == {"ok": True, "actor": None} # -- end test_actor_is_null -- @@ -258,5 +266,5 @@ async def test_signed_cookie_actor(): ds = Datasette(memory=True) cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} response = await ds.client.get("/-/actor.json", cookies=cookies) - assert response.json() == {"actor": {"id": "root"}} + assert response.json() == {"ok": True, "actor": {"id": "root"}} # -- end test_signed_cookie_actor -- diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index 613160ac..4a0014b4 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -1,10 +1,11 @@ # fmt: off # -- start datasette_with_plugin_fixture -- -from datasette import hookimpl -from datasette.app import Datasette import pytest import pytest_asyncio +from datasette import hookimpl +from datasette.app import Datasette + @pytest_asyncio.fixture async def datasette_with_plugin(): diff --git a/tests/test_error_shape.py b/tests/test_error_shape.py new file mode 100644 index 00000000..94c9a7c9 --- /dev/null +++ b/tests/test_error_shape.py @@ -0,0 +1,751 @@ +""" +Tests for the canonical JSON error shape. + +Every JSON error response from Datasette should use one shape: + + { + "ok": false, + "error": "", + "errors": ["", ...], + "status": + } + +Additional context keys (for example "rows" and "truncated" on SQL errors) +are permitted, but "ok", "error", "errors" and "status" must always be +present and the legacy "title" key must not be. + +https://github.com/simonw/datasette/issues - 1.0 API consistency +""" + +import time + +import pytest + +from datasette.app import Datasette +from datasette.utils import sqlite3 + + +def assert_canonical_error(response, expected_status): + assert response.status_code == expected_status + data = response.json() + assert data["ok"] is False + assert isinstance(data["error"], str) + assert data["error"] + assert isinstance(data["errors"], list) + assert data["errors"] + assert all(isinstance(message, str) for message in data["errors"]) + assert data["error"] == "; ".join(data["errors"]) + assert data["status"] == expected_status + assert "title" not in data + return data + + +@pytest.fixture +def ds_error_shape(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette([db_path]) + ds.root_enabled = True + yield ds + ds.close() + + +# Shape 1: the exception handler (handle_exception.py) + + +@pytest.mark.asyncio +async def test_not_found_error_shape(ds_client): + response = await ds_client.get("/fixtures/no_such_table.json") + assert_canonical_error(response, 404) + + +@pytest.mark.asyncio +async def test_datasette_error_with_title_omits_title_key(ds_client): + # DatasetteError(title="Invalid SQL") previously leaked a "title" key + response = await ds_client.get( + "/fixtures/-/query.json?sql=update+facetable+set+state+=+1" + ) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Statement must be a SELECT"] + + +# Shape 2: the _error() helper (views/base.py) - write API and friends + + +@pytest.mark.asyncio +async def test_write_api_validation_error_shape(ds_error_shape): + token = "dstok_{}".format( + ds_error_shape.sign( + {"a": "root", "token": "dstok", "t": 0}, + namespace="token", + ) + ) + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + json={"rows": [{"nope": 1}, {"also_nope": 2}]}, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + data = assert_canonical_error(response, 400) + # Multiple messages: errors keeps them all, error joins them + assert len(data["errors"]) == 2 + assert data["errors"][0].startswith("Row 0") + assert data["errors"][1].startswith("Row 1") + + +@pytest.mark.asyncio +async def test_write_api_permission_denied_shape(ds_error_shape): + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + json={"rows": [{"title": "hello"}]}, + headers={"Content-Type": "application/json"}, + ) + assert_canonical_error(response, 403) + + +# Shape 3: the JSON renderer (renderer.py) + + +@pytest.mark.asyncio +async def test_sql_error_shape_keeps_context_keys(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?sql=select+*+from+no_such_table" + ) + data = assert_canonical_error(response, 400) + # Renderer errors keep their context keys + assert data["rows"] == [] + assert "truncated" in data + + +@pytest.mark.asyncio +async def test_invalid_shape_error_shape(ds_client): + response = await ds_client.get("/fixtures/-/query.json?sql=select+1&_shape=bananas") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Invalid _shape: bananas"] + + +@pytest.mark.asyncio +async def test_shape_object_on_query_is_a_400_error(ds_client): + # Previously returned HTTP 200 with an ok: false body + response = await ds_client.get("/fixtures/-/query.json?sql=select+1&_shape=object") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["_shape=object is only available on tables"] + + +# Shape 4: bare {"error": ...} from the permission debug endpoints + + +@pytest.mark.asyncio +async def test_allowed_missing_action_error_shape(ds_client): + response = await ds_client.get("/-/allowed.json") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["action parameter is required"] + + +@pytest.mark.asyncio +async def test_allowed_unknown_action_error_shape(ds_client): + response = await ds_client.get("/-/allowed.json?action=no_such_action") + assert_canonical_error(response, 404) + + +@pytest.mark.asyncio +async def test_check_unknown_action_error_shape(ds_error_shape): + response = await ds_error_shape.client.get( + "/-/check.json?action=no_such_action", + actor={"id": "root"}, + ) + assert_canonical_error(response, 404) + + +@pytest.mark.asyncio +async def test_rules_missing_action_error_shape(ds_error_shape): + response = await ds_error_shape.client.get( + "/-/rules.json", + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["action parameter is required"] + + +# Other stragglers + + +@pytest.mark.asyncio +async def test_method_not_allowed_error_shape(ds_client): + response = await ds_client.post("/fixtures.json") + assert_canonical_error(response, 405) + + +@pytest.mark.asyncio +async def test_schema_unknown_database_error_shape(ds_client): + response = await ds_client.get("/no_such_db/-/schema.json") + assert_canonical_error(response, 404) + + +# Forbidden responses (the default forbidden() hook) + + +@pytest.fixture +def ds_forbidden(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette( + [db_path], + config={"databases": {"data": {"tables": {"docs": {"allow": {"id": "root"}}}}}}, + ) + ds.root_enabled = True + yield ds + ds.close() + + +@pytest.mark.asyncio +async def test_forbidden_json_path_returns_canonical_json(ds_forbidden): + response = await ds_forbidden.client.get("/data/docs.json") + data = assert_canonical_error(response, 403) + assert "permission" in data["error"].lower() + + +@pytest.mark.asyncio +async def test_forbidden_accept_json_returns_canonical_json(ds_forbidden): + response = await ds_forbidden.client.get( + "/data/docs", headers={"Accept": "application/json"} + ) + assert_canonical_error(response, 403) + + +@pytest.mark.asyncio +async def test_forbidden_html_path_still_returns_html(ds_forbidden): + response = await ds_forbidden.client.get("/data/docs") + assert response.status_code == 403 + assert response.headers["content-type"].startswith("text/html") + + +@pytest.mark.asyncio +async def test_forbidden_json_path_allowed_actor_still_works(ds_forbidden): + response = await ds_forbidden.client.get("/data/docs.json", actor={"id": "root"}) + assert response.status_code == 200 + assert response.json()["ok"] is True + + +# Write canned queries: SQL failures must not return HTTP 200 + + +@pytest.fixture +def ds_write_query(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette( + [db_path], + config={ + "databases": { + "data": { + "queries": { + "add_doc": { + "sql": ( + "insert into docs (id, title)" " values (:id, :title)" + ), + "write": True, + }, + "add_doc_custom_error": { + "sql": ( + "insert into docs (id, title)" " values (:id, :title)" + ), + "write": True, + "on_error_message": "Custom error message", + "on_error_redirect": "/data", + }, + } + } + } + }, + ) + yield ds + ds.close() + + +@pytest.mark.asyncio +async def test_write_query_success_returns_200(ds_write_query): + response = await ds_write_query.client.post( + "/data/add_doc", + json={"id": 1, "title": "One"}, + headers={"Accept": "application/json"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["message"] == "Query executed, 1 row affected" + assert data["redirect"] is None + + +@pytest.mark.asyncio +async def test_write_query_sql_failure_returns_400(ds_write_query): + for _ in range(2): + response = await ds_write_query.client.post( + "/data/add_doc", + json={"id": 1, "title": "One"}, + headers={"Accept": "application/json"}, + ) + data = assert_canonical_error(response, 400) + assert "UNIQUE constraint failed" in data["error"] + # The redirect context key from the canned query flow is preserved + assert data["redirect"] is None + + +@pytest.mark.asyncio +async def test_write_query_failure_uses_on_error_message_and_redirect( + ds_write_query, +): + for _ in range(2): + response = await ds_write_query.client.post( + "/data/add_doc_custom_error", + json={"id": 1, "title": "One"}, + headers={"Accept": "application/json"}, + ) + data = assert_canonical_error(response, 400) + assert data["error"] == "Custom error message" + assert data["redirect"] == "/data" + + +@pytest.mark.asyncio +async def test_write_query_forbidden_is_canonical_403(ds_write_query): + # An untrusted write query run by an actor without execute-write-sql + # raises Forbidden, handled by the forbidden() hook + await ds_write_query.invoke_startup() + await ds_write_query.add_query( + "data", + name="untrusted_add", + sql="insert into docs (id, title) values (:id, :title)", + is_write=True, + is_trusted=False, + source="user", + owner_id="someone", + ) + response = await ds_write_query.client.post( + "/data/untrusted_add", + json={"id": 5, "title": "Five"}, + headers={"Accept": "application/json"}, + actor={"id": "someone"}, + ) + assert_canonical_error(response, 403) + + +@pytest.mark.asyncio +async def test_write_query_rejected_operation_is_canonical_403(ds_write_query): + # A rejected operation (VACUUM) raises QueryWriteRejected, handled by + # the dedicated branch in QueryView.post - root has execute-write-sql + ds_write_query.root_enabled = True + await ds_write_query.invoke_startup() + await ds_write_query.add_query( + "data", + name="vacuum_it", + sql="vacuum", + is_write=True, + is_trusted=False, + source="user", + owner_id="root", + ) + response = await ds_write_query.client.post( + "/data/vacuum_it", + json={}, + headers={"Accept": "application/json"}, + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 403) + assert data["redirect"] is None + + +# Row delete write failures must be 400, matching row update + + +@pytest.mark.asyncio +async def test_row_delete_write_failure_is_400(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.execute("insert into docs (id, title) values (1, 'One')") + conn.execute( + "create trigger no_delete before delete on docs " + "begin select raise(abort, 'deletes are blocked'); end" + ) + conn.commit() + conn.close() + ds = Datasette([db_path]) + ds.root_enabled = True + try: + response = await ds.client.post( + "/data/docs/1/-/delete", + json={}, + headers={"Content-Type": "application/json"}, + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 400) + assert "deletes are blocked" in data["error"] + finally: + ds.close() + + +# Invalid bearer tokens must produce 401, not silent anonymous access + + +@pytest.mark.asyncio +async def test_expired_token_returns_401(ds_error_shape): + token = "dstok_{}".format( + ds_error_shape.sign( + {"a": "root", "t": int(time.time()) - 2000, "d": 1000}, + namespace="token", + ) + ) + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + ) + data = assert_canonical_error(response, 401) + assert "expired" in data["error"].lower() + assert response.headers["www-authenticate"].startswith("Bearer") + + +@pytest.mark.asyncio +async def test_bad_signature_token_returns_401(ds_error_shape): + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": "Bearer dstok_garbage"} + ) + assert_canonical_error(response, 401) + assert response.headers["www-authenticate"].startswith("Bearer") + + +@pytest.mark.asyncio +async def test_unrecognized_token_prefix_stays_anonymous(ds_error_shape): + # No registered handler claims this token - it might belong to a + # plugin's actor_from_request hook, so it must not hard-fail + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": "Bearer sometoken_abc"} + ) + assert response.status_code == 200 + assert response.json() == {"ok": True, "actor": None} + + +@pytest.mark.asyncio +async def test_valid_token_still_authenticates(ds_error_shape): + token = "dstok_{}".format( + ds_error_shape.sign( + {"a": "root", "t": int(time.time())}, + namespace="token", + ) + ) + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + ) + assert response.status_code == 200 + assert response.json()["actor"]["id"] == "root" + + +@pytest.mark.asyncio +async def test_bad_token_beats_valid_cookie(ds_error_shape): + # A malformed Authorization header is a hard error even if a valid + # ds_actor cookie is also present + response = await ds_error_shape.client.get( + "/-/actor.json", + headers={"Authorization": "Bearer dstok_garbage"}, + cookies={"ds_actor": ds_error_shape.client.actor_cookie({"id": "root"})}, + ) + assert_canonical_error(response, 401) + + +@pytest.mark.asyncio +async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.close() + ds = Datasette([db_path], settings={"allow_signed_tokens": False}) + try: + token = "dstok_{}".format( + ds.sign({"a": "root", "t": int(time.time())}, namespace="token") + ) + response = await ds.client.get( + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + ) + data = assert_canonical_error(response, 401) + assert "not enabled" in data["error"] + finally: + ds.close() + + +# GET /db/-/query without SQL: 400 for data formats, HTML editor stays 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/fixtures/-/query.json", + "/fixtures/-/query.json?sql=", + ), +) +async def test_query_json_without_sql_is_400(ds_client, path): + response = await ds_client.get(path) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["?sql= is required"] + + +@pytest.mark.asyncio +async def test_query_html_without_sql_is_still_the_editor(ds_client): + response = await ds_client.get("/fixtures/-/query") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + + +# Write API return:true responses use "rows" consistently + + +@pytest.mark.asyncio +async def test_row_update_return_uses_rows_list(ds_error_shape): + await ds_error_shape.client.post( + "/data/docs/-/insert", + json={"row": {"id": 1, "title": "One"}}, + headers={"Content-Type": "application/json"}, + actor={"id": "root"}, + ) + response = await ds_error_shape.client.post( + "/data/docs/1/-/update", + json={"update": {"title": "Updated"}, "return": True}, + headers={"Content-Type": "application/json"}, + actor={"id": "root"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert "row" not in data + assert data["rows"] == [{"id": 1, "title": "Updated"}] + + +# Schema endpoints: no existence oracle, no 500 on unknown database + + +@pytest.mark.asyncio +async def test_schema_endpoints_no_existence_oracle(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key)") + conn.close() + ds = Datasette([db_path], default_deny=True) + ds.root_enabled = True + try: + # An actor without view-database cannot distinguish an existing + # database from a missing one + denied_existing = await ds.client.get("/data/-/schema.json") + denied_missing = await ds.client.get("/nope/-/schema.json") + assert denied_existing.status_code == denied_missing.status_code == 403 + + # An authorized actor sees the real thing + root_existing = await ds.client.get("/data/-/schema.json", actor={"id": "root"}) + assert root_existing.status_code == 200 + root_missing = await ds.client.get("/nope/-/schema.json", actor={"id": "root"}) + assert root_missing.status_code == 404 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_table_schema_unknown_database_is_404_not_500(ds_client): + response = await ds_client.get("/no_such_db/some_table/-/schema.json") + assert_canonical_error(response, 404) + + +# Unknown _extra names are a 400, not silently ignored + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/fixtures/facetable.json?_extra=nope", + "/fixtures/facetable.json?_extra=count,nope", + "/fixtures/simple_primary_key/1.json?_extra=nope", + "/fixtures/-/query.json?sql=select+1&_extra=nope", + ), +) +async def test_unknown_extra_is_400(ds_client, path): + response = await ds_client.get(path) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Unknown _extra: nope"] + + +@pytest.mark.asyncio +async def test_html_only_extra_via_json_is_400(ds_client): + # display_rows exists for the HTML view but is not part of the JSON API + response = await ds_client.get("/fixtures/facetable.json?_extra=display_rows") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Unknown _extra: display_rows"] + + +@pytest.mark.asyncio +async def test_unknown_extra_ignored_on_html_pages(ds_client): + response = await ds_client.get("/fixtures/facetable?_extra=nope") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + + +# /-/threads exposes runtime internals and requires permissions-debug + + +@pytest.mark.asyncio +async def test_threads_requires_permissions_debug(ds_error_shape): + denied = await ds_error_shape.client.get("/-/threads.json") + assert_canonical_error(denied, 403) + allowed = await ds_error_shape.client.get("/-/threads.json", actor={"id": "root"}) + assert allowed.status_code == 200 + assert allowed.json()["ok"] is True + + +# _size is the one page-size parameter, with uniform validation + + +@pytest.mark.asyncio +async def test_query_list_size_supports_max_keyword(ds_client): + response = await ds_client.get("/fixtures/-/queries.json?_size=max") + assert response.status_code == 200 + # ds_client runs with max_returned_rows=100 + assert response.json()["limit"] == 100 + + +@pytest.mark.asyncio +async def test_query_list_size_rejects_out_of_range(ds_client): + response = await ds_client.get("/fixtures/-/queries.json?_size=5000") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["_size must be <= 100"] + + +@pytest.mark.asyncio +async def test_query_list_size_rejects_non_integer(ds_client): + response = await ds_client.get("/fixtures/-/queries.json?_size=bananas") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["_size must be a positive integer"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ("allowed", "rules")) +async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint): + base = f"/-/{endpoint}.json?action=view-instance" + ok = await ds_error_shape.client.get( + base + "&_size=1&_page=1", actor={"id": "root"} + ) + assert ok.status_code == 200 + assert ok.json()["page_size"] == 1 + + max_size = await ds_error_shape.client.get( + base + "&_size=max", actor={"id": "root"} + ) + assert max_size.status_code == 200 + assert max_size.json()["page_size"] == 200 + + too_big = await ds_error_shape.client.get(base + "&_size=500", actor={"id": "root"}) + data = assert_canonical_error(too_big, 400) + assert data["errors"] == ["_size must be <= 200"] + + bad_page = await ds_error_shape.client.get(base + "&_page=0", actor={"id": "root"}) + data = assert_canonical_error(bad_page, 400) + assert data["errors"] == ["_page must be a positive integer"] + + +# Write endpoints parse the body as JSON regardless of Content-Type + + +@pytest.mark.asyncio +async def test_insert_works_without_content_type_header(ds_error_shape): + # Previously a 500 AttributeError + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + content='{"row": {"id": 1, "title": "One"}}', + actor={"id": "root"}, + ) + assert response.status_code == 201 + assert response.json()["rows"][0]["title"] == "One" + + +@pytest.mark.asyncio +async def test_insert_works_with_form_content_type(ds_error_shape): + # Previously 400 "Invalid content-type, must be application/json" + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + content='{"row": {"id": 2, "title": "Two"}}', + headers={"Content-Type": "application/x-www-form-urlencoded"}, + actor={"id": "root"}, + ) + assert response.status_code == 201 + + +@pytest.mark.asyncio +async def test_insert_form_encoded_body_is_invalid_json(ds_error_shape): + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + content="title=Three", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 400) + assert data["errors"][0].startswith("Invalid JSON:") + + +@pytest.mark.asyncio +async def test_alter_and_set_column_type_ignore_content_type(ds_error_shape): + alter = await ds_error_shape.client.post( + "/data/docs/-/alter", + content='{"operations": [{"op": "add_column", "args": {"name": "extra"}}]}', + actor={"id": "root"}, + ) + assert alter.status_code == 200, alter.text + sct = await ds_error_shape.client.post( + "/data/docs/-/set-column-type", + content='{"column": "title", "column_type": {"type": "textarea"}}', + actor={"id": "root"}, + ) + assert sct.status_code == 200, sct.text + + +# SQL Interrupted errors carry plain text in JSON, not an HTML fragment + + +@pytest.mark.asyncio +async def test_sql_interrupted_json_error_is_plain_text(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5" + ) + data = assert_canonical_error(response, 400) + assert "<" not in data["error"] + assert data["error"].startswith("SQL query took too long.") + + +@pytest.mark.asyncio +async def test_sql_interrupted_html_page_keeps_rich_error(ds_client): + response = await ds_client.get( + "/fixtures/-/query?sql=select+sleep(0.01)&_timelimit=5" + ) + assert response.status_code == 400 + assert "      '.format( - i, i - ), + f'', f'', f'', f'', @@ -760,6 +834,39 @@ async def test_table_html_foreign_key_links(ds_client): ] +@pytest.mark.asyncio +@pytest.mark.parametrize("referenced_table", ("authors", "AuThOrS")) +async def test_table_html_foreign_key_to_missing_table_is_not_linked(referenced_table): + # https://github.com/simonw/datasette/issues/1515 + ds = Datasette([]) + db = ds.add_database( + Database(ds, memory_name="test_foreign_key_to_missing_table"), name="data" + ) + await db.execute_write_script(f""" + create table authors (id integer primary key, name text); + create table books ( + id integer primary key, + author_id integer references {referenced_table}(id), + missing_id integer references missing_table(id) + ); + insert into authors (id, name) values (1, 'Ada'); + insert into books (id, author_id, missing_id) values (1, 1, 7); + """) + response = await ds.client.get("/data/books") + assert response.status_code == 200 + table = Soup(response.text, "html.parser").find("table") + cells = {td["class"][0]: str(td) for td in table.select("tbody tr")[0].select("td")} + assert cells["col-author_id"] == ( + '' + ) + assert cells["col-missing_id"] == '' + # The JSON labels are left alone as well + data = (await ds.client.get("/data/books.json?_labels=on")).json() + assert data["rows"][0]["missing_id"] == 7 + assert data["rows"][0]["author_id"] == {"value": 1, "label": "Ada"} + + @pytest.mark.asyncio async def test_table_html_foreign_key_facets(ds_client): response = await ds_client.get( @@ -1027,6 +1134,7 @@ async def test_database_create_table_action_button_and_data(): "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], "defaultExpressions": DEFAULT_EXPRESSION_OPTIONS, + "canInsertRows": False, }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] @@ -1050,6 +1158,40 @@ async def test_database_create_table_action_button_and_data(): ds.close() +@pytest.mark.asyncio +async def test_database_create_table_data_includes_insert_row_permission(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + "insert-row": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_insert_permission"), + name="data", + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[ + "createTable" + ] + assert create_table_data["canInsertRows"] is True + finally: + ds.close() + + @pytest.mark.asyncio async def test_database_create_table_data_includes_custom_column_types(): ds = Datasette( @@ -1316,6 +1458,7 @@ async def test_table_insert_action_button_and_data(): assert insert_data["path"] == "/data/items/-/insert" assert insert_data["tableName"] == "items" assert insert_data["primaryKeys"] == ["id"] + assert insert_data["maxInsertRows"] == 100 assert [column["name"] for column in insert_data["columns"]] == [ "name", "score", @@ -1629,9 +1772,9 @@ async def test_row_update_sets_message(): json={"update": {"name": long_name}, "return": True}, ) assert response.status_code == 200 - assert response.json()["row"]["name"] == long_name + assert response.json()["rows"][0]["name"] == long_name assert ds.unsign(response.cookies["ds_messages"], "messages") == [ - ["Updated row 1 ({})".format(truncated_name), ds.INFO] + [f"Updated row 1 ({truncated_name})", ds.INFO] ] finally: ds.close() @@ -1644,9 +1787,9 @@ def test_table_data_uses_base_url(app_client_base_url_prefix): import re soup = Soup(response.text, "html.parser") - table_script = [ + table_script = next( s for s in soup.find_all("script") if "_datasetteTableData" in (s.string or "") - ][0] + ) match = re.search( r"window\._datasetteTableData\s*=\s*({.*?});", table_script.string, @@ -1674,16 +1817,17 @@ def test_table_fragment_custom_table_include(): @pytest.mark.asyncio async def test_table_fragment_uses_render_cell_hook(): - from datasette import hookimpl from markupsafe import Markup + from datasette import hookimpl + class TestRenderCellPlugin: __name__ = "TestRenderCellPlugin" @hookimpl def render_cell(self, value, column, table, database): if database == "data" and table == "items" and column == "name": - return Markup("{}".format(value)) + return Markup(f"{value}") return None ds = Datasette(memory=True) @@ -1979,8 +2123,8 @@ async def test_sort_errors(ds_client, json, params, error): assert response.json() == { "ok": False, "error": error, + "errors": [error], "status": 400, - "title": None, } else: assert error in response.text @@ -2222,18 +2366,16 @@ def test_allow_facet_off(allow_facet): ) async def test_format_of_binary_links(size, title, length_bytes): ds = Datasette() - db_name = "binary-links-{}".format(size) + db_name = f"binary-links-{size}" db = ds.add_memory_database(db_name) - sql = "select zeroblob({}) as blob".format(size) - await db.execute_write("create table blobs as {}".format(sql)) - response = await ds.client.get("/{}/blobs".format(db_name)) + sql = f"select zeroblob({size}) as blob" + await db.execute_write(f"create table blobs as {sql}") + response = await ds.client.get(f"/{db_name}/blobs") assert response.status_code == 200 - expected = "{}><Binary: {} bytes>".format(title, length_bytes) + expected = f"{title}><Binary: {length_bytes} bytes>" assert expected in response.text # And test with arbitrary SQL query too - sql_response = await ds.client.get( - "{}/-/query".format(db_name), params={"sql": sql} - ) + sql_response = await ds.client.get(f"{db_name}/-/query", params={"sql": sql}) assert sql_response.status_code == 200 assert expected in sql_response.text @@ -2313,6 +2455,7 @@ async def test_foreign_key_labels_obey_permissions(config): assert root_b.json() == { "ok": True, "next": None, + "next_url": None, "rows": [{"id": 1, "name": "world", "a_id": {"value": 1, "label": "hello"}}], "truncated": False, } @@ -2320,6 +2463,7 @@ async def test_foreign_key_labels_obey_permissions(config): assert anon_b.json() == { "ok": True, "next": None, + "next_url": None, "rows": [{"id": 1, "name": "world", "a_id": 1}], "truncated": False, } diff --git a/tests/test_table_resource_identity.py b/tests/test_table_resource_identity.py new file mode 100644 index 00000000..35b4dcc6 --- /dev/null +++ b/tests/test_table_resource_identity.py @@ -0,0 +1,420 @@ +"""Table permission identities must agree with SQLite identifier resolution.""" + +import uuid +from unittest.mock import AsyncMock + +import pytest + +from datasette import hookimpl +from datasette.app import Datasette +from datasette.default_permissions import restrictions_allow_action +from datasette.permissions import Action, PermissionSQL, _permission_check_cache +from datasette.resources import QueryResource, TableResource +from datasette.utils.actions_sql import explain_permission_for_resource +from datasette.utils.asgi import Forbidden +from datasette.utils.permissions import gather_permission_sql_from_hooks + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["table", "view"]) +@pytest.mark.parametrize("spelling", ["Inventory", "inventory", "INVENTORY"]) +@pytest.mark.parametrize("allowed", [False, True]) +@pytest.mark.parametrize("rule_spelling", ["Inventory", "iNvEnToRy"]) +async def test_table_permission_identity( + kind, spelling, allowed, rule_spelling, monkeypatch +): + ds = Datasette( + config={ + "permissions": {"view-table": not allowed, "insert-row": not allowed}, + "databases": { + "data": { + "tables": { + rule_spelling: { + "permissions": { + "view-table": allowed, + "insert-row": allowed, + } + } + } + } + }, + } + ) + db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + cache_token = _permission_check_cache.set({}) + try: + await db.execute_write( + "create table Inventory (id integer primary key)" + if kind == "table" + else "create view Inventory as select 1 as id" + ) + await ds.invoke_startup() + # Identity matching needs no target-schema lookup. Derived-table + # permissions may still check the schema version. All spellings and + # API entry points should share the existing permission result cache. + target_execute = AsyncMock(wraps=db.execute) + monkeypatch.setattr(db, "execute", target_execute) + internal_execute = AsyncMock(wraps=ds.get_internal_database().execute) + monkeypatch.setattr(ds.get_internal_database(), "execute", internal_execute) + resource = TableResource("data", spelling) + assert await ds.allowed_many( + actions=["view-table", "insert-row"], resource=resource + ) == {"view-table": allowed, "insert-row": allowed} + assert await ds.allowed(action="view-table", resource=resource) is allowed + assert await ds.check_visibility(None, "view-table", resource) == ( + allowed, + False, + ) + if allowed: + await ds.ensure_permission(action="view-table", resource=resource) + else: + with pytest.raises(Forbidden): + await ds.ensure_permission(action="view-table", resource=resource) + assert resource.child == spelling # Do not mutate caller-owned resources. + for variant in ("Inventory", "inventory", "INVENTORY"): + assert ( + await ds.allowed( + action="view-table", resource=TableResource("data", variant) + ) + is allowed + ) + assert internal_execute.await_count == 1 + assert all( + call.args[0] == "PRAGMA schema_version" + for call in target_execute.await_args_list + ) + assert all(key[3] == "inventory" for key in _permission_check_cache.get()) + finally: + _permission_check_cache.reset(cache_token) + ds.close() + + +@pytest.mark.asyncio +async def test_other_permission_identities_are_preserved(): + ds = Datasette( + config={ + "databases": { + "data": { + "tables": { + "Äpfel": {"permissions": {"view-table": False}}, + "Future": {"permissions": {"view-table": False}}, + }, + "queries": { + "Report": { + "sql": "select 1", + "permissions": {"view-query": False}, + }, + "report": { + "sql": "select 1", + "permissions": {"view-query": True}, + }, + }, + } + } + } + ) + db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + try: + await db.execute_write('create table "Äpfel" (id integer primary key)') + await db.execute_write('create table "äpfel" (id integer primary key)') + await db.execute_write("create table Report (id integer primary key)") + await ds.invoke_startup() + # SQLite folds ASCII identifier casing, not Unicode casing. + for name, expected in [ + ("ÄPFEL", False), + ("äPFEL", True), + ("Future", False), + ("future", False), + ]: + assert ( + await ds.allowed( + action="view-table", resource=TableResource("data", name) + ) + is expected + ) + # Query names remain case-sensitive even when a table has the same name. + for name, expected in [("Report", False), ("report", True)]: + assert ( + await ds.allowed( + action="view-query", resource=QueryResource("data", name) + ) + is expected + ) + finally: + ds.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allow", [True, False, {"id": "reader"}]) +async def test_table_listings_and_explanations(allow): + ds = Datasette( + config={ + "databases": { + "data": { + "tables": { + "inventory": {"permissions": {"view-table": allow}}, + } + } + } + } + ) + db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + try: + await db.execute_write("create table Inventory (id integer primary key)") + await db.execute_write("create view InventoryView as select id from Inventory") + await ds.invoke_startup() + for actor in (None, {"id": "reader"}): + expected = allow is True or (isinstance(allow, dict) and actor == allow) + explanation = await explain_permission_for_resource( + datasette=ds, + actor=actor, + action="view-table", + parent="data", + child="INVENTORY", + ) + assert explanation["allowed"] is expected + assert explanation["winning_scope"] == "resource" + assert any( + "data/inventory" in rule["reason"] + for rule in explanation["matched_rules"] + ) + page = await ds.allowed_resources( + "view-table", + actor, + parent="data", + include_is_private=True, + include_reasons=True, + limit=1, + ) + resources = [resource async for resource in page.all()] + matching = [r for r in resources if r.child == "Inventory"] + assert bool(matching) is expected + assert len(matching) <= 1 + if matching: + assert matching[0].private is isinstance(allow, dict) + assert any(r.child == "InventoryView" for r in resources) + finally: + ds.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_first", [True, False]) +async def test_case_variant_rules_deny_wins(deny_first): + rules = [("inventory", False), ("INVENTORY", True)] + if not deny_first: + rules.reverse() + ds = Datasette( + config={ + "databases": { + "data": { + "tables": { + name: {"permissions": {"view-table": allow}} + for name, allow in rules + } + } + } + } + ) + db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + try: + await db.execute_write("create table Inventory (id integer primary key)") + await ds.invoke_startup() + assert not await ds.allowed( + action="view-table", resource=TableResource("data", "Inventory") + ) + assert not ( + await ds.allowed_resources( + "view-table", parent="data", include_is_private=True + ) + ).resources + explanation = await explain_permission_for_resource( + datasette=ds, + actor=None, + action="view-table", + parent="data", + child="Inventory", + ) + assert not explanation["allowed"] + assert any( + rule["effect"] == "allow" and not rule["decisive"] + for rule in explanation["matched_rules"] + ) + assert any( + rule["effect"] == "deny" and rule["decisive"] + for rule in explanation["matched_rules"] + ) + finally: + ds.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("config_style", ["allow", "permissions"]) +@pytest.mark.parametrize("allowed", [True, False]) +async def test_case_variant_token_restrictions(config_style, allowed): + table_config = ( + {"allow": allowed} + if config_style == "allow" + else {"permissions": {"view-table": allowed}} + ) + ds = Datasette( + config={"databases": {"data": {"tables": {"Inventory": table_config}}}} + ) + db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + actor = {"id": "reader", "_r": {"r": {"data": {"inventory": ["vt"]}}}} + try: + await db.execute_write("create table Inventory (id integer primary key)") + await ds.invoke_startup() + assert restrictions_allow_action( + ds, actor["_r"], "view-table", ("data", "INVENTORY") + ) + assert not restrictions_allow_action( + ds, actor["_r"], "view-table", ("Data", "Inventory") + ) + assert ( + await ds.allowed( + action="view-table", + resource=TableResource("data", "INVENTORY"), + actor=actor, + ) + is allowed + ) + page = await ds.allowed_resources("view-table", actor, parent="data") + assert [(r.parent, r.child) for r in page.resources] == ( + [("data", "Inventory")] if allowed else [] + ) + explanation = await explain_permission_for_resource( + datasette=ds, + actor=actor, + action="view-table", + parent="data", + child="Inventory", + ) + assert explanation["restriction_allowed"] + assert explanation["allowed"] is allowed + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_plugin_restriction_intersection_and_dependencies(): + class Plugin: + @hookimpl + def register_actions(self, datasette): + return [ + Action( + name="inspect-inventory", + description="Inspect inventory", + resource_class=TableResource, + also_requires="view-table", + ) + ] + + @hookimpl + def permission_resources_sql(self, action): + if action not in ("view-table", "inspect-inventory"): + return None + return [ + PermissionSQL( + sql="SELECT 'data' AS parent, 'INVENTORY' AS child, 1 AS allow, 'inventory grant' AS reason", + restriction_sql="SELECT 'data' AS parent, 'inventory' AS child", + ), + PermissionSQL( + restriction_sql="SELECT 'data' AS parent, 'InVeNtOrY' AS child" + ), + ] + + ds = Datasette(default_deny=True) + ds.pm.register(Plugin(), name="identity-test") + db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + try: + await db.execute_write("create table Inventory (id integer primary key)") + await db.execute_write("create table Other (id integer primary key)") + await ds.invoke_startup() + for action in ("view-table", "inspect-inventory"): + assert await ds.allowed( + action=action, resource=TableResource("data", "Inventory") + ) + assert not await ds.allowed( + action=action, resource=TableResource("data", "Other") + ) + resources = ( + await ds.allowed_resources( + action, parent="data", include_is_private=True + ) + ).resources + assert [r.child for r in resources] == ["Inventory"] + explanation = await explain_permission_for_resource( + datasette=ds, + actor=None, + action=action, + parent="data", + child="Inventory", + ) + assert explanation["allowed"] + assert all(item["allowed"] for item in explanation["restrictions"]) + finally: + ds.pm.unregister(name="identity-test") + ds.close() + + +@pytest.mark.asyncio +async def test_shared_plugin_rule_keeps_query_identity_and_original_sql(): + shared = PermissionSQL( + sql="SELECT 'data' AS parent, 'Inventory' AS child, 0 AS allow, 'shared deny' AS reason" + ) + original_sql = shared.sql + + class Plugin: + @hookimpl + def permission_resources_sql(self, action): + if action in ("view-table", "view-query"): + return shared + + ds = Datasette( + config={ + "databases": { + "data": { + "queries": { + "Inventory": "select 1", + "inventory": "select 1", + } + } + } + } + ) + ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") + ds.pm.register(Plugin(), name="identity-test") + try: + await ds.invoke_startup() + for _ in range(2): + await gather_permission_sql_from_hooks( + datasette=ds, actor=None, action="view-table" + ) + assert shared.sql == original_sql + assert not await ds.allowed( + action="view-table", resource=TableResource("data", "inventory") + ) + assert await ds.allowed( + action="view-query", resource=QueryResource("data", "inventory") + ) + assert not await ds.allowed( + action="view-query", resource=QueryResource("data", "Inventory") + ) + assert await ds.allowed( + action="view-table", resource=TableResource("Data", "Inventory") + ) + assert restrictions_allow_action( + ds, + {"r": {"data": {"Inventory": ["vq"]}}}, + "view-query", + ("data", "Inventory"), + ) + assert not restrictions_allow_action( + ds, + {"r": {"data": {"Inventory": ["vq"]}}}, + "view-query", + ("data", "inventory"), + ) + finally: + ds.pm.unregister(name="identity-test") + ds.close() diff --git a/tests/test_tasks_endpoint.py b/tests/test_tasks_endpoint.py new file mode 100644 index 00000000..be174dbe --- /dev/null +++ b/tests/test_tasks_endpoint.py @@ -0,0 +1,122 @@ +""" +Tests for the /-/tasks introspection endpoint. + +/-/tasks exposes datasette._background_tasks (see tests/test_background_tasks.py +for the supervisor machinery itself) the same way /-/threads exposes threading +internals: gated behind the permissions-debug permission, JSON-only. +""" + +import asyncio +import contextlib +import functools + +import pytest + +from datasette.app import Datasette + + +async def example_task(datasette): + pass + + +class ExampleWorker: + async def run(self, datasette): + pass + + async def __call__(self, datasette): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "func, qualified_name", + [ + (example_task, "example_task"), + (functools.partial(example_task), "example_task"), + (ExampleWorker().run, "ExampleWorker.run"), + (ExampleWorker(), "ExampleWorker.__call__"), + ], +) +async def test_task_function_path(func, qualified_name): + ds = Datasette(memory=True) + ds.root_enabled = True + handle = ds.add_background_task(func, name="custom-name") + try: + response = await ds.client.get("/-/tasks.json", actor={"id": "root"}) + assert response.status_code == 200 + task = response.json()["tasks"][0] + assert task["name"] == "custom-name" + assert task["function"] == f"{__name__}.{qualified_name}" + assert handle.function == task["function"] + assert "plugin" not in task + await handle.task + html = await ds.client.get("/-/tasks", actor={"id": "root"}) + assert html.status_code == 200 + assert task["function"] in html.text + finally: + await ds.invoke_shutdown() + + +@pytest.mark.asyncio +async def test_tasks_requires_permissions_debug(): + ds = Datasette(memory=True) + ds.root_enabled = True + + denied = await ds.client.get("/-/tasks.json") + assert denied.status_code == 403 + + allowed = await ds.client.get("/-/tasks.json", actor={"id": "root"}) + assert allowed.status_code == 200 + data = allowed.json() + assert data["ok"] is True + assert "tasks" in data + assert "launched" in data + + +@pytest.mark.asyncio +async def test_running_and_crashed_task_states(): + ds = Datasette(memory=True) + ds.root_enabled = True + + async def long_running(datasette): + await asyncio.Event().wait() + + async def crashing_task(datasette): + raise RuntimeError("kaboom") + + long_handle = ds.add_background_task(long_running, name="long-runner") + crash_handle = ds.add_background_task(crashing_task, name="crashing_task") + + await ds.start_background_tasks() + + # Let the crashing_task run to completion and its done-callback (which sets + # handle.state = "crashed") actually fire before we read state back out. + await asyncio.wait_for( + asyncio.gather(crash_handle.task, return_exceptions=True), timeout=5 + ) + await asyncio.sleep(0) + + try: + response = await ds.client.get("/-/tasks.json", actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["launched"] is True + + by_name = {t["name"]: t for t in data["tasks"]} + assert by_name["long-runner"]["state"] == "running" + assert by_name["long-runner"]["exception"] is None + assert by_name["long-runner"]["started_at"] is not None + + crashed = by_name["crashing_task"] + assert crashed["function"] == ( + f"{__name__}.test_running_and_crashed_task_states..crashing_task" + ) + assert crashed["state"] == "crashed" + assert crashed["exception"] is not None + assert isinstance(crashed["exception"], str) + assert "kaboom" in crashed["exception"] + assert "RuntimeError" in crashed["exception"] + finally: + long_handle.cancel() + with contextlib.suppress(asyncio.CancelledError): + await long_handle.task diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 00000000..801876df --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,1023 @@ +import json +import sqlite3 +import subprocess +import sys +import threading +import time + +import pytest +import sqlite_utils +from opentelemetry import context as otel_context_api +from opentelemetry import trace as otel_trace +from opentelemetry.trace import SpanKind, StatusCode + +from datasette.app import Datasette +from datasette.database import Database, QueryInterrupted +from datasette.telemetry import ( + MAX_SQL_LENGTH, + SCHEMA_URL, + sql_attribute, + sql_operation_name, + tracer, +) +from datasette.version import __version__ + +SECRET_PARAM_VALUE = "SUPER_SECRET_PARAM_VALUE_XYZ_123" + +INVALID_SQL = "select this_is_not_valid_sql from nowhere" + +# Bounded so a broken time limit fails rather than hangs, but too slow to +# finish within the millisecond time limits used below. +SLOW_SQL = """ +with recursive counter(x) as ( + select 1 union all select x + 1 from counter where x < 50000000 +) +select max(x) from counter +""" + + +def _db_query_spans(otel_spans): + return [span for span in otel_spans.get_finished_spans() if span.name == "db.query"] + + +def _spans_for_namespace(otel_spans, namespace): + "db.query spans for one database, excluding queries against the internal database." + return [ + span + for span in _db_query_spans(otel_spans) + if span.attributes["db.namespace"] == namespace + ] + + +def _children_named(otel_spans, name, parent_span_context): + "Finished spans called `name` that are direct children of `parent_span_context`." + return [ + span + for span in otel_spans.get_finished_spans() + if span.name == name + and span.parent is not None + and span.parent.span_id == parent_span_context.span_id + and span.parent.trace_id == parent_span_context.trace_id + and span.context.trace_id == parent_span_context.trace_id + ] + + +def _descends_from(span, ancestor_span_context, by_span_id): + "True if `span` reaches `ancestor_span_context` by walking parent links." + seen = set() + current = span + while current.parent is not None: + if current.parent.span_id == ancestor_span_context.span_id: + return current.parent.trace_id == ancestor_span_context.trace_id + if current.parent.span_id in seen: + return False + seen.add(current.parent.span_id) + current = by_span_id.get(current.parent.span_id) + if current is None: + return False + return False + + +def _all_attribute_values(otel_spans): + "Every attribute value on every finished span and span event." + values = [] + for span in otel_spans.get_finished_spans(): + values.extend((span.attributes or {}).values()) + for event in span.events: + values.extend((event.attributes or {}).values()) + return values + + +def test_datasette_package_never_imports_the_sdk(): + """ + Importing datasette does not load the OpenTelemetry SDK. + + conftest.py moves this test to the front of the run by name. + """ + code = ( + "import datasette.app, datasette.database, datasette.telemetry, sys; " + "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert ( + result.stdout.strip() == "[]" + ), f"datasette imported the OpenTelemetry SDK: {result.stdout.strip()}" + + +@pytest.mark.asyncio +async def test_db_query_span_basic_attributes(ds_client, otel_spans): + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans, "expected at least one db.query span" + span = spans[-1] + + assert span.attributes["db.system"] == "sqlite" + assert span.attributes["db.namespace"] == "fixtures" + assert span.attributes["db.query.text"] == "select 1" + assert span.attributes["datasette.rows_returned"] == 1 + assert span.attributes["datasette.truncated"] is False + assert isinstance(span.attributes["datasette.time_limit_ms"], int) + assert span.status.status_code == StatusCode.UNSET + + +@pytest.mark.asyncio +async def test_truncated_result_sets_truncated_attribute(otel_spans): + "A result cut short by max_returned_rows records truncated=True." + ds = Datasette(memory=True, settings={"max_returned_rows": 5}) + db = ds.add_memory_database("t04_truncated") + results = await db.execute( + "select value from json_each('[1,2,3,4,5,6,7,8,9,10]')", truncate=True + ) + assert results.truncated + + spans = _spans_for_namespace(otel_spans, "t04_truncated") + assert spans + span = spans[-1] + assert span.attributes["datasette.truncated"] is True + assert span.attributes["datasette.rows_returned"] == 5 + + +@pytest.mark.asyncio +async def test_facetable_request_produces_db_query_spans(ds_client, otel_spans): + response = await ds_client.get("/fixtures/facetable.json") + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans, "expected at least one db.query span" + assert all(span.attributes["db.system"] == "sqlite" for span in spans) + # Each span records the SQL or, for callback methods, the callback name: + assert all( + span.attributes.get("db.query.text") + or span.attributes.get("datasette.callback") + for span in spans + ) + assert any(span.attributes.get("db.query.text") for span in spans) + # Rendering the page also queries the internal database, so only some of + # these spans belong to "fixtures". + assert any(span.attributes["db.namespace"] == "fixtures" for span in spans) + + +def test_sql_attribute_truncates_at_2048(): + short_sql = "select 1" + assert sql_attribute(short_sql) == "select 1" + # Surrounding whitespace is stripped: + assert sql_attribute(" select 1\n") == "select 1" + + long_sql = "select 1 -- " + ("x" * 3000) + truncated = sql_attribute(long_sql) + assert len(truncated) == MAX_SQL_LENGTH + len("…[truncated]") + assert truncated.startswith("select 1 -- ") + assert truncated.endswith("…[truncated]") + + +@pytest.mark.asyncio +async def test_db_query_text_is_truncated_in_real_span(ds_client, otel_spans): + # A long trailing comment keeps the SQL valid but over the 2048 character limit + long_sql = "select 1 -- " + ("x" * 3000) + response = await ds_client.get("/fixtures/-/query.json", params={"sql": long_sql}) + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans + assert any(len(span.attributes.get("db.query.text", "")) > 100 for span in spans), ( + "expected the long query to reach a span - otherwise this test would " + "pass even if truncation were never applied" + ) + for span in spans: + recorded = span.attributes.get("db.query.text", "") + assert len(recorded) <= MAX_SQL_LENGTH + len("…[truncated]") + + +@pytest.mark.asyncio +async def test_no_span_attribute_ever_contains_a_parameter_value(ds_client, otel_spans): + response = await ds_client.get( + "/fixtures/-/query.json", + params={"sql": "select :secret", "secret": SECRET_PARAM_VALUE}, + ) + assert response.status_code == 200 + # Confirm the bound parameter value was used by the query: + assert SECRET_PARAM_VALUE in json.dumps(response.json()) + + for value in _all_attribute_values(otel_spans): + if isinstance(value, str): + assert SECRET_PARAM_VALUE not in value + elif isinstance(value, (list, tuple)): + for item in value: + if isinstance(item, str): + assert SECRET_PARAM_VALUE not in item + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert "select :secret" in span.attributes["db.query.text"] + assert span.attributes.get("datasette.param_count") == 1 + + +@pytest.mark.asyncio +async def test_query_interrupted_sets_error_status(otel_spans): + """ + A query that exceeds the sql_time_limit_ms setting is a span error. + + The limit comes from the setting because a shorter custom_time_limit + marks the timeout as expected. + """ + ds = Datasette(memory=True, settings={"sql_time_limit_ms": 20}) + db = ds.add_memory_database("t09_instance_limit_timeout") + with pytest.raises(QueryInterrupted): + await db.execute(SLOW_SQL) + + spans = _spans_for_namespace(otel_spans, "t09_instance_limit_timeout") + assert spans + span = spans[-1] + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["datasette.interrupted"] is True + assert span.events + assert all(event.name == "exception" for event in span.events) + + +async def _expected_timeout_count_span(otel_spans, database_name): + "Make table_counts() time out and return its db.query span." + db = Datasette(memory=True).add_memory_database(database_name) + await db.execute_write("create table big (id integer primary key, t text)") + await db.execute_write_many( + "insert into big (t) values (?)", [["x" * 50] for _ in range(11000)] + ) + # count_limit caps the scan at 10001 rows. Below 20ms sqlite_timelimit() + # checks the limit on every VM instruction, so this reliably exceeds 1ms. + counts = await db.table_counts(1) + assert counts == { + "big": None + }, "the count did not actually time out, so the rest of this test is vacuous" + + spans = [ + span + for span in _spans_for_namespace(otel_spans, database_name) + if "count(*)" in span.attributes.get("db.query.text", "") + ] + assert len(spans) == 1 + return spans[0] + + +@pytest.mark.asyncio +async def test_expected_timeout_is_not_a_span_error(otel_spans): + span = await _expected_timeout_count_span(otel_spans, "t09_expected_timeout") + # Recorded as interrupted, but not as an error: + assert span.attributes["datasette.interrupted"] is True + assert span.status.status_code != StatusCode.ERROR + assert not [event for event in span.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_expected_timeout_does_not_error_the_inner_execute_span(otel_spans): + "The db.query.execute child span is not marked as an error either." + span = await _expected_timeout_count_span(otel_spans, "t09_expected_timeout_inner") + children = _children_named(otel_spans, "db.query.execute", span.context) + assert len(children) == 1 + child = children[0] + assert child.status.status_code != StatusCode.ERROR + assert not [event for event in child.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_unexpected_timeout_is_still_a_span_error(otel_spans): + "A timeout is an error if custom_time_limit is above sql_time_limit_ms." + ds = Datasette(memory=True, settings={"sql_time_limit_ms": 20}) + db = ds.add_memory_database("t09_custom_limit_ignored") + with pytest.raises(QueryInterrupted): + await db.execute(SLOW_SQL, custom_time_limit=5000) + + spans = _spans_for_namespace(otel_spans, "t09_custom_limit_ignored") + assert spans + span = spans[-1] + # The setting overrides the larger custom_time_limit: + assert span.attributes["datasette.time_limit_ms"] == 20 + assert span.attributes["datasette.interrupted"] is True + assert span.status.status_code == StatusCode.ERROR + assert any(event.name == "exception" for event in span.events) + + children = _children_named(otel_spans, "db.query.execute", span.context) + assert len(children) == 1 + assert children[0].status.status_code == StatusCode.ERROR + + +@pytest.mark.asyncio +async def test_unsuppressed_sql_error_is_a_span_error(ds_client, otel_spans): + db = ds_client.ds.get_database("fixtures") + with pytest.raises(sqlite3.OperationalError): + await db.execute(INVALID_SQL) + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert span.status.status_code == StatusCode.ERROR + assert any(event.name == "exception" for event in span.events) + assert "datasette.sql_error_suppressed" not in span.attributes + + +@pytest.mark.asyncio +async def test_suppressed_sql_error_is_not_a_span_error(ds_client, otel_spans): + "With log_sql_errors=False the error is recorded as suppressed, not a span error." + db = ds_client.ds.get_database("fixtures") + with pytest.raises(sqlite3.OperationalError): + await db.execute(INVALID_SQL, log_sql_errors=False) + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert span.status.status_code == StatusCode.UNSET + assert span.attributes["datasette.sql_error_suppressed"] is True + assert not [event for event in span.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_execute_write_produces_db_query_span(otel_spans): + # Named in-memory databases are shared, so each test uses a unique name. + db = Datasette(memory=True).add_memory_database("t03_write_span") + await db.execute_write("create table docs (id integer primary key, name text)") + await db.execute_write("insert into docs (id, name) values (?, ?)", [1, "one"]) + + spans = _spans_for_namespace(otel_spans, "t03_write_span") + assert spans, "expected db.query spans from execute_write()" + span = spans[-1] + + assert span.attributes["db.system"] == "sqlite" + assert span.attributes["db.namespace"] == "t03_write_span" + assert span.attributes["db.query.text"] == ( + "insert into docs (id, name) values (?, ?)" + ) + assert span.attributes["datasette.param_count"] == 2 + + +@pytest.mark.asyncio +async def test_execute_write_script_sets_executescript_attribute(otel_spans): + db = Datasette(memory=True).add_memory_database("t03_write_script_span") + await db.execute_write_script( + "create table docs (id integer primary key);\n" + "insert into docs (id) values (1);" + ) + + spans = _spans_for_namespace(otel_spans, "t03_write_script_span") + assert spans, "expected a db.query span from execute_write_script()" + span = spans[-1] + + assert span.attributes["db.system"] == "sqlite" + assert span.attributes["datasette.executescript"] is True + assert "insert into docs" in span.attributes["db.query.text"] + + +@pytest.mark.asyncio +async def test_execute_write_many_records_param_sets_not_rows_returned(otel_spans): + db = Datasette(memory=True).add_memory_database("t03_write_many_span") + await db.execute_write("create table docs (id integer primary key)") + await db.execute_write_many( + "insert into docs (id) values (?)", [[i] for i in range(1, 6)] + ) + + spans = _spans_for_namespace(otel_spans, "t03_write_many_span") + many_spans = [ + span for span in spans if span.attributes.get("datasette.executemany") is True + ] + assert len(many_spans) == 1 + span = many_spans[0] + + assert span.attributes["datasette.param_sets"] == 5 + assert "datasette.rows_returned" not in span.attributes + + +# --- Context propagation across thread boundaries -------------------------- +# +# These tests check span parentage, not just that the spans exist. + + +@pytest.mark.asyncio +async def test_db_query_execute_parents_to_db_query(ds_client, otel_spans): + # execute_fn() submits to the executor, so db.query.execute is created on + # another thread. + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + query_spans = [ + span + for span in _spans_for_namespace(otel_spans, "fixtures") + if span.attributes.get("db.query.text") == "select 1" + ] + assert query_spans, "expected a db.query span for 'select 1'" + query_span = query_spans[-1] + + assert [ + span + for span in otel_spans.get_finished_spans() + if span.name == "db.query.execute" + ], "expected at least one db.query.execute span" + children = _children_named(otel_spans, "db.query.execute", query_span.context) + assert len(children) == 1, "expected exactly one db.query.execute child of db.query" + # db.query.execute runs within db.query; the gap is the thread pool wait. + assert query_span.start_time <= children[0].start_time + assert children[0].end_time <= query_span.end_time + + +@pytest.mark.asyncio +async def test_immutable_database_propagates_context(tmp_path, otel_spans): + # Immutable databases run execute_isolated_fn() on another thread using + # loop.run_in_executor(), not the write thread. + db_path = tmp_path / "t04_immutable.db" + sqlite_utils.Database(str(db_path))["t"].insert({"id": 1}, pk="id") + + ds = Datasette() + db = Database(ds, path=str(db_path), is_mutable=False) + ds.add_database(db, name="t04_immutable") + + def fn(conn): + with tracer.start_as_current_span("t04-child-in-isolated-worker"): + pass + + try: + with tracer.start_as_current_span("t04-parent-on-event-loop") as parent: + parent_context = parent.get_span_context() + await db.execute_isolated_fn(fn) + finally: + ds.remove_database("t04_immutable") + + assert [ + span + for span in otel_spans.get_finished_spans() + if span.name == "t04-child-in-isolated-worker" + ], "expected a span created inside execute_isolated_fn's worker thread" + # Expected chain: event loop parent -> db.query -> worker thread child + query_spans = _children_named(otel_spans, "db.query", parent_context) + assert len(query_spans) == 1 + children = _children_named( + otel_spans, "t04-child-in-isolated-worker", query_spans[0].context + ) + assert len(children) == 1 + + +@pytest.mark.asyncio +async def test_write_spans_parent_to_db_query(otel_spans): + # execute_write() queues a WriteTask for the write thread. + # db.write.queue_wait and db.write.execute are both children of db.query. + db = Datasette(memory=True).add_memory_database("t04_write_spans") + await db.execute_write("create table docs (id integer primary key)") + + query_spans = _spans_for_namespace(otel_spans, "t04_write_spans") + assert query_spans, "expected a db.query span from execute_write()" + query_span = query_spans[-1] + + queue_wait_children = _children_named( + otel_spans, "db.write.queue_wait", query_span.context + ) + execute_children = _children_named( + otel_spans, "db.write.execute", query_span.context + ) + assert len(queue_wait_children) == 1 + assert len(execute_children) == 1 + + execute_span = execute_children[0] + assert execute_span.attributes["datasette.isolated_connection"] is False + assert execute_span.attributes["datasette.transaction"] is True + # The queue wait ends before the write begins. + assert queue_wait_children[0].end_time <= execute_span.start_time + + +@pytest.mark.asyncio +async def test_write_queue_wait_duration_reflects_real_wait(otel_spans): + # db.write.queue_wait runs from task.enqueued_at_ns, captured on the event + # loop, to when the write thread dequeues the task. + ds = Datasette(memory=True) + db = ds.add_memory_database("t04_queue_wait") + await db.execute_write("create table docs (id integer primary key)") + + def slow_write(conn): + time.sleep(0.1) + + # Queue a slow write without waiting for it, then a second write behind it: + _, slow_future = await db._send_to_write_thread(slow_write, block=False) + await db.execute_write("insert into docs (id) values (1)") + await slow_future + + query_spans = [ + span + for span in _spans_for_namespace(otel_spans, "t04_queue_wait") + if span.attributes.get("db.query.text") == "insert into docs (id) values (1)" + ] + assert query_spans, "expected a db.query span for the queued-behind insert" + queue_wait_children = _children_named( + otel_spans, "db.write.queue_wait", query_spans[-1].context + ) + assert len(queue_wait_children) == 1 + duration_ns = queue_wait_children[0].end_time - queue_wait_children[0].start_time + # The slow write sleeps for 100ms + assert duration_ns > 10_000_000, f"queue wait was only {duration_ns}ns" + + +async def _write_spans_from_one_enqueue(otel_spans, name, block): + """ + Run one write through the write thread inside a span, returning + (enqueueing span context, {span name: span}). + + Uses _send_to_write_thread() because execute_write() would add its own + db.query span between the enqueueing span and the write spans. + """ + db = Datasette(memory=True).add_memory_database(name) + await db.execute_write("create table docs (id integer primary key)") + + def insert(conn): + conn.execute("insert into docs (id) values (1)") + + otel_spans.clear() + with tracer.start_as_current_span("enqueueing-span") as enqueuer: + enqueuer_context = enqueuer.get_span_context() + queued = await db._send_to_write_thread(insert, block=block) + if not block: + # Wait for the write after the enqueueing span has ended. The reply + # future resolves once both write spans have been exported. + _, reply_future = queued + await reply_future + + spans = {} + for span in otel_spans.get_finished_spans(): + if span.name in ("db.write.queue_wait", "db.write.execute"): + assert span.name not in spans, f"more than one {span.name} span" + spans[span.name] = span + assert set(spans) == {"db.write.queue_wait", "db.write.execute"} + return enqueuer_context, spans + + +@pytest.mark.asyncio +async def test_blocking_write_spans_still_parent_normally(otel_spans): + # block=True waits for the write, so its spans are children of the + # enqueueing span, with no links. + enqueuer_context, spans = await _write_spans_from_one_enqueue( + otel_spans, "t07_blocking_write", block=True + ) + for name, span in spans.items(): + assert span.parent is not None, f"{name} lost its parent" + assert span.parent.span_id == enqueuer_context.span_id, name + assert span.parent.trace_id == enqueuer_context.trace_id, name + assert span.context.trace_id == enqueuer_context.trace_id, name + assert span.links == (), f"{name} should be parented, not linked" + + +@pytest.mark.asyncio +async def test_nonblocking_write_spans_are_roots_with_a_link(otel_spans): + # block=False returns before the write runs, so the write spans are roots + # linked to the enqueueing span. + enqueuer_context, spans = await _write_spans_from_one_enqueue( + otel_spans, "t07_nonblocking_write", block=False + ) + assert enqueuer_context.is_valid, "test's own enqueueing span was not recorded" + for name, span in spans.items(): + assert span.parent is None, f"{name} is still parented" + # Each write span starts its own trace + assert span.context.trace_id != enqueuer_context.trace_id, name + assert len(span.links) == 1, f"{name} has links {span.links}" + link_context = span.links[0].context + assert link_context.trace_id == enqueuer_context.trace_id, name + assert link_context.span_id == enqueuer_context.span_id, name + # The two write spans are separate roots + assert ( + spans["db.write.queue_wait"].context.trace_id + != spans["db.write.execute"].context.trace_id + ) + + +@pytest.mark.asyncio +async def test_nonblocking_write_link_has_no_attributes(otel_spans): + _, spans = await _write_spans_from_one_enqueue( + otel_spans, "t07_nonblocking_link_attrs", block=False + ) + for name, span in spans.items(): + assert len(span.links) == 1, name + assert dict(span.links[0].attributes or {}) == {}, name + + +@pytest.mark.asyncio +async def test_nonblocking_write_spans_ignore_the_write_threads_ambient_context( + otel_spans, +): + """ + block=False spans ignore any context left attached on the write thread. + + A prepare_connection hook could attach a context and never detach it. + This test does that, then checks the write spans are still roots. + """ + ds = Datasette(memory=True) + db = ds.add_memory_database("t07_ambient_write_thread") + write_thread_name = "_execute_writes for database t07_ambient_write_thread" + real_prepare_connection = ds._prepare_connection + leaked = {} + + def prepare_connection(conn, database): + if threading.current_thread().name == write_thread_name: + # Runs on the write thread before any task is dequeued, and never + # detaches. + span = tracer.start_span("leaked-write-thread-ambient-span") + leaked["span_id"] = span.get_span_context().span_id + otel_context_api.attach(otel_trace.set_span_in_context(span)) + return real_prepare_connection(conn, database) + + ds._prepare_connection = prepare_connection + try: + await db.execute_write("create table docs (id integer primary key)") + + def insert(conn): + conn.execute("insert into docs (id) values (1)") + + otel_spans.clear() + with tracer.start_as_current_span("enqueueing-span") as enqueuer: + enqueuer_context = enqueuer.get_span_context() + _, reply_future = await db._send_to_write_thread(insert, block=False) + await reply_future + finally: + ds._prepare_connection = real_prepare_connection + db.close() + + assert "span_id" in leaked, "the ambient context was never leaked - test is vacuous" + write_spans = [ + span + for span in otel_spans.get_finished_spans() + if span.name in ("db.write.queue_wait", "db.write.execute") + ] + assert len(write_spans) == 2 + for span in write_spans: + assert span.parent is None, ( + f"{span.name} parented to the write thread's leftover ambient " + "context instead of being a root" + ) + assert span.links[0].context.span_id == enqueuer_context.span_id + + +@pytest.mark.asyncio +async def test_suppressed_error_does_not_mark_execute_span(ds_client, otel_spans): + "The inner db.query.execute span also respects log_sql_errors=False." + db = ds_client.ds.get_database("fixtures") + with pytest.raises(sqlite3.OperationalError): + await db.execute(INVALID_SQL, log_sql_errors=False) + + execute_spans = [ + span + for span in otel_spans.get_finished_spans() + if span.name == "db.query.execute" + ] + assert execute_spans + span = execute_spans[-1] + assert span.status.status_code == StatusCode.UNSET + assert not [event for event in span.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_invoke_startup_produces_one_trace_not_dozens_of_orphans(otel_spans): + "Spans emitted by invoke_startup() share a single datasette.startup root span." + ds = Datasette(memory=True) + ds.add_memory_database("t05_startup_db") + # Ignore spans from constructing Datasette, which happens before startup + otel_spans.clear() + + # No ambient span, as in the ASGI lifespan path where startup runs before + # any request. + assert ( + not otel_trace.get_current_span().get_span_context().is_valid + ), "this test must run with no ambient span" + + await ds.invoke_startup() + + spans = otel_spans.get_finished_spans() + assert len(spans) > 10, f"expected startup to emit many spans, got {len(spans)}" + + startup_spans = [span for span in spans if span.name == "datasette.startup"] + assert len(startup_spans) == 1 + startup = startup_spans[0] + assert startup.parent is None, "datasette.startup should be a root span" + + trace_ids = {span.context.trace_id for span in spans} + assert trace_ids == {startup.context.trace_id}, ( + f"startup produced {len(trace_ids)} distinct traces; every span it " + "causes should share the datasette.startup trace" + ) + + roots = [span for span in spans if span.parent is None] + assert [span.name for span in roots] == ["datasette.startup"] + + by_span_id = {span.context.span_id: span for span in spans} + + # Internal database reads: + internal_queries = [ + span + for span in spans + if span.name == "db.query" and span.attributes["db.namespace"] == "__INTERNAL__" + ] + assert internal_queries, "expected internal-catalog db.query spans during startup" + assert all( + _descends_from(span, startup.context, by_span_id) for span in internal_queries + ) + + # Internal database writes, which run on the write thread: + write_spans = [span for span in spans if span.name.startswith("db.write.")] + assert write_spans, "expected db.write.* spans during startup" + assert all( + _descends_from(span, startup.context, by_span_id) for span in write_spans + ) + + +# --- Semantic conventions: span kind, scope, db.operation.name ------------- + + +@pytest.mark.asyncio +async def test_db_query_is_client_kind_and_children_are_internal(otel_spans): + """ + db.query spans are CLIENT. Their child spans are INTERNAL because they are + parts of one query rather than separate database calls. + """ + db = Datasette(memory=True).add_memory_database("t06_span_kind") + # Call each of the four SQL string methods: + await db.execute_write("create table docs (id integer primary key)") + await db.execute_write_many( + "insert into docs (id) values (?)", [[i] for i in range(1, 4)] + ) + await db.execute_write_script("insert into docs (id) values (99);") + await db.execute("select id from docs") + + query_spans = _spans_for_namespace(otel_spans, "t06_span_kind") + assert len(query_spans) == 4, "expected a db.query span per entry point" + for span in query_spans: + text = span.attributes["db.query.text"] + assert span.kind == SpanKind.CLIENT, f"db.query for {text!r} should be CLIENT" + + for name in ("db.query.execute", "db.write.execute", "db.write.queue_wait"): + children = [ + span for span in otel_spans.get_finished_spans() if span.name == name + ] + assert children, f"expected at least one {name} span" + for span in children: + assert span.kind == SpanKind.INTERNAL, f"{name} should be INTERNAL" + + +@pytest.mark.asyncio +async def test_instrumentation_scope_declares_version_and_schema_url( + ds_client, otel_spans +): + "The instrumentation scope includes the Datasette version and schema URL." + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans, "expected at least one db.query span" + scope = spans[-1].instrumentation_scope + + assert scope.name == "datasette" + assert scope.version == __version__ + # Uses the literal URL so changing SCHEMA_URL requires updating this test + assert scope.schema_url == "https://opentelemetry.io/schemas/1.29.0" + assert SCHEMA_URL == "https://opentelemetry.io/schemas/1.29.0" + assert __version__, "the scope version must not be empty" + + +def test_db_operation_name_from_leading_keyword(): + assert sql_operation_name("select 1") == "SELECT" + assert sql_operation_name(" insert into x (a) values (1)") == "INSERT" + # A leading CTE reports WITH, not the operation inside it + assert sql_operation_name("with foo as (select 1) select * from foo") == "WITH" + # Unrecognized leading keyword + assert sql_operation_name("gibberish 1") is None + # A parenthesized SELECT or a leading comment also returns None + assert sql_operation_name("(select 1) union select 2") is None + assert sql_operation_name("-- a comment\nselect 1") is None + assert sql_operation_name("") is None + + +@pytest.mark.asyncio +async def test_db_operation_name_on_real_span(ds_client, otel_spans): + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + spans = [ + span + for span in _spans_for_namespace(otel_spans, "fixtures") + if span.attributes.get("db.query.text") == "select 1" + ] + assert spans, "expected a db.query span for 'select 1'" + assert spans[-1].attributes["db.operation.name"] == "SELECT" + + +@pytest.mark.asyncio +async def test_execute_write_sets_db_operation_name(otel_spans): + db = Datasette(memory=True).add_memory_database("t06_write_operation") + await db.execute_write("create table docs (id integer primary key)") + await db.execute_write_many( + "insert into docs (id) values (?)", [[i] for i in range(1, 4)] + ) + + spans = _spans_for_namespace(otel_spans, "t06_write_operation") + by_operation = { + span.attributes["db.query.text"]: span.attributes.get("db.operation.name") + for span in spans + } + assert by_operation["create table docs (id integer primary key)"] == "CREATE" + assert by_operation["insert into docs (id) values (?)"] == "INSERT" + + +@pytest.mark.asyncio +async def test_execute_write_script_has_no_operation_name(otel_spans): + """ + Scripts can contain several statements, so db.operation.name is omitted. + + The script starts with `create`, which is on the allowlist, so this fails + if the operation name is extracted anyway. + """ + db = Datasette(memory=True).add_memory_database("t06_script_operation") + await db.execute_write_script( + "create table docs (id integer primary key);\n" + "insert into docs (id) values (1);" + ) + + spans = _spans_for_namespace(otel_spans, "t06_script_operation") + script_spans = [ + span for span in spans if span.attributes.get("datasette.executescript") is True + ] + assert len(script_spans) == 1 + assert "db.operation.name" not in script_spans[0].attributes + + +# --- Callback-style calls: execute_fn / execute_write_fn / execute_isolated_fn + + +@pytest.mark.asyncio +async def test_execute_fn_produces_db_query_span(otel_spans): + db = Datasette(memory=True).add_memory_database("t16_execute_fn") + await db.execute_write("create table t (id integer primary key)") + + def count_rows(conn): + return conn.execute("select count(*) from t").fetchone()[0] + + otel_spans.clear() + assert await db.execute_fn(count_rows) == 0 + + spans = _spans_for_namespace(otel_spans, "t16_execute_fn") + assert len(spans) == 1 + span = spans[0] + assert span.kind == SpanKind.CLIENT + assert span.attributes["db.system"] == "sqlite" + assert ( + span.attributes["datasette.callback"] + == "test_execute_fn_produces_db_query_span..count_rows" + ) + # Callbacks have no SQL text to record or take an operation name from + assert "db.query.text" not in span.attributes + assert "db.operation.name" not in span.attributes + children = _children_named(otel_spans, "db.query.execute", span.context) + assert len(children) == 1 + + +@pytest.mark.asyncio +async def test_execute_fn_lambda_reports_lambda(otel_spans): + db = Datasette(memory=True).add_memory_database("t16_lambda") + otel_spans.clear() + await db.execute_fn(lambda conn: conn.execute("select 1").fetchone()) + spans = _spans_for_namespace(otel_spans, "t16_lambda") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"].endswith("") + + +@pytest.mark.asyncio +async def test_execute_write_fn_produces_db_query_span(otel_spans): + db = Datasette(memory=True).add_memory_database("t16_write_fn") + + def create_table(conn): + conn.execute("create table t (id integer primary key)") + + otel_spans.clear() + await db.execute_write_fn(create_table) + + spans = _spans_for_namespace(otel_spans, "t16_write_fn") + assert len(spans) == 1 + span = spans[0] + assert span.kind == SpanKind.CLIENT + assert ( + span.attributes["datasette.callback"] + == "test_execute_write_fn_produces_db_query_span..create_table" + ) + assert "db.query.text" not in span.attributes + # The write-thread spans are this span's children, same as execute_write() + for name in ("db.write.queue_wait", "db.write.execute"): + assert len(_children_named(otel_spans, name, span.context)) == 1, name + + +@pytest.mark.asyncio +async def test_execute_write_fn_callback_name_is_not_the_hook_wrapper(otel_spans): + # _wrap_fn_with_hooks() wraps callbacks that accept track_event + db = Datasette(memory=True).add_memory_database("t16_wrapper_name") + + def create_with_events(conn, track_event): + conn.execute("create table t (id integer primary key)") + + otel_spans.clear() + await db.execute_write_fn(create_with_events) + spans = _spans_for_namespace(otel_spans, "t16_wrapper_name") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"] == ( + "test_execute_write_fn_callback_name_is_not_the_hook_wrapper" + "..create_with_events" + ) + + +@pytest.mark.asyncio +async def test_execute_write_fn_nonblocking_spans_link_to_the_new_span(otel_spans): + # With block=False the write thread spans link to the db.query span from + # execute_write_fn(), not to the span that was current when it was called. + db = Datasette(memory=True).add_memory_database("t16_nonblocking") + await db.execute_write("create table docs (id integer primary key)") + + def insert(conn): + conn.execute("insert into docs (id) values (1)") + + otel_spans.clear() + with tracer.start_as_current_span("t16-enqueueing-span") as enqueuer: + enqueuer_context = enqueuer.get_span_context() + await db.execute_write_fn(insert, block=False) + # Writes run in order, so this waits for the non-blocking write to finish + await db.execute_write("insert into docs (id) values (2)") + + query_spans = [ + span + for span in _spans_for_namespace(otel_spans, "t16_nonblocking") + if span.attributes.get("datasette.callback") + ] + assert len(query_spans) == 1 + fn_span_context = query_spans[0].context + linked = [ + span + for span in otel_spans.get_finished_spans() + if span.name in ("db.write.queue_wait", "db.write.execute") and span.links + ] + assert len(linked) == 2 + for span in linked: + assert span.parent is None, f"{span.name} is still parented" + assert span.links[0].context.span_id == fn_span_context.span_id, span.name + assert span.links[0].context.span_id != enqueuer_context.span_id, span.name + + +@pytest.mark.asyncio +async def test_execute_does_not_double_wrap(otel_spans): + # execute() and the SQL string write methods call the private + # _execute_fn() and _execute_write_fn(), so they create one db.query span. + db = Datasette(memory=True).add_memory_database("t16_no_double_wrap") + otel_spans.clear() + await db.execute_write("create table t (id integer primary key)") + assert len(_spans_for_namespace(otel_spans, "t16_no_double_wrap")) == 1 + otel_spans.clear() + await db.execute("select * from t") + spans = _spans_for_namespace(otel_spans, "t16_no_double_wrap") + assert len(spans) == 1 + assert len(_children_named(otel_spans, "db.query.execute", spans[0].context)) == 1 + + +@pytest.mark.asyncio +async def test_execute_isolated_fn_span_on_mutable_and_immutable(tmp_path, otel_spans): + def read_one(conn): + return conn.execute("select 1").fetchone()[0] + + mutable = Datasette(memory=True).add_memory_database("t16_isolated_mutable") + otel_spans.clear() + assert await mutable.execute_isolated_fn(read_one) == 1 + spans = _spans_for_namespace(otel_spans, "t16_isolated_mutable") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"].endswith("read_one") + # Mutable databases route through the write thread, so the write spans + # appear as children; immutable ones run on the pool and get none. + assert _children_named(otel_spans, "db.write.execute", spans[0].context) + + db_path = tmp_path / "t16_isolated_immutable.db" + sqlite_utils.Database(str(db_path))["t"].insert({"id": 1}) + ds = Datasette() + immutable = Database(ds, path=str(db_path), is_mutable=False) + ds.add_database(immutable, name="t16_isolated_immutable") + try: + otel_spans.clear() + assert await immutable.execute_isolated_fn(read_one) == 1 + finally: + ds.remove_database("t16_isolated_immutable") + spans = _spans_for_namespace(otel_spans, "t16_isolated_immutable") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"].endswith("read_one") + assert not _children_named(otel_spans, "db.write.execute", spans[0].context) + + +@pytest.mark.asyncio +async def test_execute_fn_exception_marks_span_error(otel_spans): + # execute_fn() has no log_sql_errors option, so exceptions are span errors + db = Datasette(memory=True).add_memory_database("t16_fn_error") + + def boom(conn): + raise ValueError("callback failed") + + otel_spans.clear() + with pytest.raises(ValueError): + await db.execute_fn(boom) + spans = _spans_for_namespace(otel_spans, "t16_fn_error") + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + assert any(event.name == "exception" for event in spans[0].events) diff --git a/tests/test_telemetry_metrics.py b/tests/test_telemetry_metrics.py new file mode 100644 index 00000000..2e465b82 --- /dev/null +++ b/tests/test_telemetry_metrics.py @@ -0,0 +1,483 @@ +""" +Tests for the OpenTelemetry metrics emitted by Datasette. Gauge callbacks are +called directly, since the pool gauges have no attributes to tell instances apart. +""" + +import asyncio +import threading +import weakref + +import pytest + +from datasette import telemetry +from datasette.app import Datasette +from datasette.database import Database +from datasette.utils.sqlite import sqlite3 + +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + + +def observations(callback, datasette=None): + """ + Run a gauge callback, optionally keeping only observations produced by one + Datasette's databases. Returns a list of (attributes dict, value). + """ + results = [] + names = None + if datasette is not None: + names = {db.name for db in telemetry._databases_of(datasette)} + for observation in callback(): + attributes = dict(observation.attributes or {}) + namespace = attributes.get("db.namespace") + if names is not None and namespace is not None and namespace not in names: + continue + results.append((attributes, observation.value)) + return results + + +@pytest.fixture +def metrics_ds(): + "A Datasette with a distinctive thread count and a uniquely named database." + ds = Datasette( + memory=True, + settings={"num_sql_threads": 7}, + ) + ds.add_memory_database("metrics_test_db") + try: + yield ds + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_sql_thread_limit_gauge_reports_num_sql_threads(metrics_ds): + values = [value for _, value in observations(telemetry.observe_sql_thread_limit)] + # Other Datasette instances may also be reporting: + assert 7 in values + + +@pytest.mark.asyncio +async def test_no_thread_gauges_in_non_threaded_mode(): + "Pool gauges skip instances with num_sql_threads=0, which have no pool." + ds = Datasette(memory=True, settings={"num_sql_threads": 0}) + try: + assert ds.executor is None + # Pool gauges have no attributes, so observe only this instance: + original = telemetry._live_datasettes + telemetry._live_datasettes = weakref.WeakSet([ds]) + try: + assert list(telemetry.observe_sql_thread_limit()) == [] + assert list(telemetry.observe_sql_thread_queue_depth()) == [] + finally: + telemetry._live_datasettes = original + # Per-database gauges do not depend on the pool: + assert observations(telemetry.observe_pending_queries, ds) + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_thread_queue_depth_gauge_reports_saturation(): + """ + Queue depth is above zero when reads queue behind num_sql_threads. Also + fails if the private ThreadPoolExecutor._work_queue attribute goes away. + """ + ds = Datasette(memory=True, settings={"num_sql_threads": 1}) + db = ds.add_memory_database("metrics_saturation_db") + entered = threading.Event() + release = threading.Event() + + def blocker(conn): + entered.set() + assert release.wait(timeout=10) + return 1 + + try: + first = asyncio.ensure_future(db.execute_fn(blocker)) + # Wait until the blocker is using the only thread: + await asyncio.get_running_loop().run_in_executor(None, entered.wait, 10) + second = asyncio.ensure_future(db.execute_fn(lambda conn: 2)) + # The second query is queued on a later event loop turn, so poll: + depths = [] + for _ in range(500): + depths = [ + value + for _, value in observations(telemetry.observe_sql_thread_queue_depth) + ] + if any(value >= 1 for value in depths): + break + await asyncio.sleep(0.01) + assert any(value >= 1 for value in depths), depths + release.set() + assert await first == 1 + assert await second == 2 + finally: + release.set() + ds.close() + + +@pytest.mark.asyncio +async def test_pending_queries_gauge_tracks_in_flight_queries(metrics_ds): + db = metrics_ds.get_database("metrics_test_db") + attributes = {"db.namespace": "metrics_test_db"} + + def value(): + points = [ + v + for a, v in observations(telemetry.observe_pending_queries, metrics_ds) + if a == attributes + ] + assert len(points) == 1 + return points[0] + + assert value() == 0 + + # Hold the worker thread until release is set: + release = asyncio.Event() + loop = asyncio.get_running_loop() + entered = asyncio.Event() + + def blocking_fn(conn): + loop.call_soon_threadsafe(entered.set) + asyncio.run_coroutine_threadsafe(release.wait(), loop).result() + return "done" + + task = asyncio.ensure_future(db.execute_fn(blocking_fn)) + await entered.wait() + assert value() == 1, "a query occupying a pool thread must be counted as pending" + release.set() + assert await task == "done" + assert value() == 0, "the count must drop once the query completes" + + +@pytest.mark.asyncio +async def test_write_queue_depth_gauge(metrics_ds): + db = metrics_ds.get_database("metrics_test_db") + attributes = {"db.namespace": "metrics_test_db"} + + def depths(): + return [ + v + for a, v in observations(telemetry.observe_write_queue_depth, metrics_ds) + if a == attributes + ] + + # No observation until the write queue has been created: + assert depths() == [] + + await db.execute_write("create table t (id integer primary key)") + assert depths() == [0], "an idle write queue reports zero, not nothing" + + +@pytest.mark.asyncio +async def test_open_connections_gauge(metrics_ds, tmp_path): + path = str(tmp_path / "conns.db") + sqlite3.connect(path).execute("create table t (id integer primary key)") + db = metrics_ds.add_database(Database(metrics_ds, path=path), name="conns_db") + attributes = {"db.namespace": "conns_db"} + + def open_connections(): + points = [ + v + for a, v in observations(telemetry.observe_open_connections, metrics_ds) + if a == attributes + ] + assert len(points) == 1 + return points[0] + + assert open_connections() == 0 + await db.execute("select 1") + assert open_connections() >= 1, "executing a query opens a tracked connection" + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_read(otel_metrics): + ds = Datasette(memory=True) + ds.add_memory_database("duration_read_db") + try: + db = ds.get_database("duration_read_db") + await db.execute("select 1") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_read_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert point.sum > 0 + assert dict(point.attributes)["db.system"] == "sqlite" + assert "error.type" not in dict(point.attributes) + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_write(otel_metrics): + ds = Datasette(memory=True) + ds.add_memory_database("duration_write_db") + try: + db = ds.get_database("duration_write_db") + await db.execute_write("create table t (id integer primary key)") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_write_db", "datasette.operation": "write"}, + ) + assert point.count == 1 + assert point.sum > 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_records_error_type(otel_metrics): + "A failed query is still timed, and is separable from a successful one." + ds = Datasette(memory=True) + ds.add_memory_database("duration_error_db") + try: + db = ds.get_database("duration_error_db") + with pytest.raises(sqlite3.OperationalError): + await db.execute("select * from nope") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_error_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert dict(point.attributes)["error.type"] == "OperationalError" + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_records_write_error_type(otel_metrics): + "A failed write is still timed and records error.type." + ds = Datasette(memory=True) + ds.add_memory_database("duration_write_error_db") + try: + db = ds.get_database("duration_write_error_db") + with pytest.raises(sqlite3.OperationalError): + await db.execute_write("insert into nope values (1)") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_write_error_db", "datasette.operation": "write"}, + ) + assert point.count == 1 + assert dict(point.attributes)["error.type"] == "OperationalError" + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_write_queue_wait_histogram(otel_metrics): + ds = Datasette(memory=True) + ds.add_memory_database("queue_wait_db") + try: + db = ds.get_database("queue_wait_db") + await db.execute_write("create table t (id integer primary key)") + await db.execute_write("insert into t (id) values (1)") + otel_metrics.collect() + point = otel_metrics.point( + "datasette.write.queue_wait", {"db.namespace": "queue_wait_db"} + ) + assert point.count == 2, "one measurement per write dequeued" + assert point.sum >= 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_interrupted_queries_counter(otel_metrics): + "Queries cancelled by sql_time_limit_ms are counted." + ds = Datasette(memory=True, settings={"sql_time_limit_ms": 1}) + ds.add_memory_database("interrupted_db") + try: + db = ds.get_database("interrupted_db") + from datasette.database import QueryInterrupted + + with pytest.raises(QueryInterrupted): + await db.execute(""" + with recursive counter(x) as ( + select 0 union all select x + 1 from counter + ) + select * from counter + """) + otel_metrics.collect() + point = otel_metrics.point( + "datasette.sql.queries.interrupted", {"db.namespace": "interrupted_db"} + ) + assert point.value == 1 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_metrics_are_reported_through_the_sdk_for_gauges(otel_metrics): + "Gauge callbacks reach the metric reader as data points." + ds = Datasette(memory=True) + ds.add_memory_database("gauge_pipeline_db") + try: + await ds.get_database("gauge_pipeline_db").execute("select 1") + otel_metrics.collect() + point = otel_metrics.point( + "datasette.sql.queries.pending", {"db.namespace": "gauge_pipeline_db"} + ) + assert point.value == 0 + assert otel_metrics.points("datasette.sql.threads.limit") + finally: + ds.close() + + +def test_closed_datasette_stops_being_observed(): + ds = Datasette(memory=True) + ds.add_memory_database("closed_db") + assert observations(telemetry.observe_pending_queries, ds) + ds.close() + names = [ + attributes.get("db.namespace") + for attributes, _ in observations(telemetry.observe_pending_queries) + ] + assert "closed_db" not in names + + +def test_registry_holds_instances_weakly(): + """ + Registering an instance does not keep it alive. Uses a stand-in object + because an atexit handler in Database.__init__ keeps a real Datasette alive. + """ + import gc + import weakref + + class FakeDatasette: + pass + + fake = FakeDatasette() + telemetry.register_datasette(fake) + assert fake in telemetry._live_instances() + ref = weakref.ref(fake) + del fake + gc.collect() + assert ref() is None + assert not any(isinstance(ds, FakeDatasette) for ds in telemetry._live_instances()) + + +HISTOGRAM_PROBES = [ + # (instrument attribute on telemetry, metric name, isolating attributes) + ( + "sql_operation_duration", + "db.client.operation.duration", + {"db.namespace": "bucket_probe_operation"}, + ), + ( + "write_queue_wait", + "datasette.write.queue_wait", + {"db.namespace": "bucket_probe_queue_wait"}, + ), +] + +# One value in each of six registry buckets. The SDK's default boundaries +# would put the first five in the same bucket. +SPREAD = [0.00005, 0.0003, 0.002, 0.03, 0.8, 7.0] + + +@pytest.mark.parametrize( + "instrument_name,metric_name,attributes", + HISTOGRAM_PROBES, + ids=[metric for _, metric, _ in HISTOGRAM_PROBES], +) +def test_histograms_spread_values_across_buckets( + otel_metrics, instrument_name, metric_name, attributes +): + """ + The registry's bucket boundaries reach the SDK. Values are recorded + directly because real test query durations would all share one bucket. + """ + from datasette.telemetry_registry import METRICS + + metric = next(m for m in METRICS if m == metric_name) + instrument = getattr(telemetry, instrument_name) + for value in SPREAD: + instrument.record(value, attributes) + + otel_metrics.collect() + point = otel_metrics.point(metric_name, attributes) + + assert ( + tuple(point.explicit_bounds) == metric.buckets + ), "the registry's boundaries did not reach the SDK" + assert point.count == len(SPREAD) + occupied = [count for count in point.bucket_counts if count] + assert len(occupied) == len(SPREAD), ( + f"expected each of {SPREAD} in its own bucket, got bucket counts " + f"{list(point.bucket_counts)} for bounds {list(point.explicit_bounds)}" + ) + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_records_execute_fn(otel_metrics): + "execute_fn() reads are recorded in the same histogram as SQL reads." + ds = Datasette(memory=True) + ds.add_memory_database("duration_fn_db") + try: + db = ds.get_database("duration_fn_db") + + def read_one(conn): + return conn.execute("select 1").fetchone()[0] + + assert await db.execute_fn(read_one) == 1 + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_fn_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert point.sum > 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_records_execute_write_fn(otel_metrics): + "execute_write_fn() writes are recorded in the same histogram." + ds = Datasette(memory=True) + ds.add_memory_database("duration_write_fn_db") + try: + db = ds.get_database("duration_write_fn_db") + + def create_table(conn): + conn.execute("create table t (id integer primary key)") + + await db.execute_write_fn(create_table) + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_write_fn_db", "datasette.operation": "write"}, + ) + assert point.count == 1 + assert point.sum > 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_records_callback_error_type(otel_metrics): + "A callback that raises is still timed, with error.type from the exception." + ds = Datasette(memory=True) + ds.add_memory_database("duration_fn_error_db") + try: + db = ds.get_database("duration_fn_error_db") + + def boom(conn): + raise ValueError("callback failed") + + with pytest.raises(ValueError): + await db.execute_fn(boom) + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_fn_error_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert dict(point.attributes)["error.type"] == "ValueError" + finally: + ds.close() diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py new file mode 100644 index 00000000..11a66a13 --- /dev/null +++ b/tests/test_telemetry_registry.py @@ -0,0 +1,502 @@ +""" +Tests that the spans, attributes and metrics Datasette emits match +datasette/telemetry_registry.py, in both directions. +""" + +import copy +import io +import itertools +import pickle + +import pytest +import pytest_asyncio + +pytest.importorskip("opentelemetry.sdk") + +from opentelemetry.trace import SpanKind + +from datasette import hookimpl +from datasette import telemetry_registry as reg +from datasette.app import Datasette +from datasette.database import QueryInterrupted +from datasette.telemetry_testing import assert_metrics_conform, assert_metrics_covered +from datasette.utils.sqlite import sqlite3 + +# Written out as literals rather than read from the registry, so renaming a +# signal fails these tests. +EXPECTED_ATTRIBUTES = { + "db.query": { + "db.system", + "db.namespace", + "db.query.text", + "datasette.callback", + "db.operation.name", + "datasette.param_count", + "datasette.param_sets", + "datasette.time_limit_ms", + "datasette.rows_returned", + "datasette.truncated", + "datasette.interrupted", + "datasette.sql_error_suppressed", + "datasette.executescript", + "datasette.executemany", + }, + "db.query.execute": set(), + "db.write.queue_wait": set(), + "db.write.execute": { + "datasette.isolated_connection", + "datasette.transaction", + }, + "datasette.startup": set(), +} +EXPECTED_SPANS = set(EXPECTED_ATTRIBUTES) + +# The HTTP request span name is composed at runtime as "{method} {route}", so +# it is checked by shape rather than as a literal. The workload only issues GETs. +EXPECTED_HTTP_SPAN_NAME = "{http.request.method} {http.route}" +EXPECTED_HTTP_METHOD_NAMES = {"GET"} +EXPECTED_HTTP_ATTRIBUTES = { + "http.request.method", + "http.route", + "url.path", + "url.scheme", + "server.address", + "user_agent.original", + "http.response.status_code", + "error.type", + "datasette.internal_client", +} + +# The registry uses the name template for the request span. +EXPECTED_REGISTRY_ATTRIBUTES = dict( + EXPECTED_ATTRIBUTES, **{EXPECTED_HTTP_SPAN_NAME: EXPECTED_HTTP_ATTRIBUTES} +) +EXPECTED_REGISTRY_NAMES = set(EXPECTED_REGISTRY_ATTRIBUTES) + +# Named in-memory databases are shared between instances, so each workload +# uses a unique name. +_names = itertools.count() + + +def _unique(prefix): + return f"{prefix}{next(_names)}" + + +class _BoomPlugin: + "A route that raises, producing a 500 and error.type on the request span." + + __name__ = "TelemetryRegistryBoomPlugin" + + @hookimpl + def register_routes(self): + return [(r"^/-/telemetry-registry-boom$", lambda: 1 / 0)] + + +async def exercise(): + """ + Drive enough of Datasette to emit every registered span and attribute, + including datasette.startup. Returns the instance so the caller can close it. + """ + name = _unique("registry") + ds = Datasette(memory=True) + ds.add_memory_database(name) + # datasette.startup + await ds.invoke_startup() + db = ds.get_database(name) + + # Writes: db.write.queue_wait, db.write.execute, db.query + await db.execute_write("create table t (id integer primary key, v text)") + # datasette.executemany, datasette.param_sets + await db.execute_write_many( + "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(30)] + ) + # datasette.executescript + await db.execute_write_script("create table t2 (id integer); drop table t2;") + # datasette.transaction=False - VACUUM cannot run inside a transaction + await db.execute_write("vacuum", transaction=False) + # datasette.isolated_connection=True + await db.execute_isolated_fn(lambda conn: conn.execute("select 1").fetchone()) + + # datasette.callback, using named functions rather than lambdas + def registry_read_callback(conn): + return conn.execute("select count(*) from t").fetchone() + + def registry_write_callback(conn): + conn.execute("insert into t (id, v) values (100, 'callback')") + + await db.execute_fn(registry_read_callback) + await db.execute_write_fn(registry_write_callback) + + # Reads: db.query.execute, datasette.rows_returned, datasette.truncated, + # datasette.param_count, datasette.time_limit_ms + await db.execute("select * from t where id > :n", {"n": 5}) + await db.execute("select * from t", truncate=True) + + # datasette.sql_error_suppressed + with pytest.raises(sqlite3.OperationalError): + await db.execute("select nope from t", log_sql_errors=False) + + # datasette.interrupted: an unbounded recursive CTE always exceeds 1ms + with pytest.raises(QueryInterrupted): + await db.execute( + "with recursive c(x) as (select 0 union all select x+1 from c) " + "select * from c", + custom_time_limit=1, + ) + + # HTTP request spans and their attributes + assert (await ds.client.get(f"/{name}/t?_facet=v")).status_code == 200 + assert (await ds.client.get(f"/{name}/t/1.json")).status_code == 200 + + # error.type on the request span, set by a 5xx response + ds.pm.register(_BoomPlugin(), name="telemetry-registry-boom") + try: + response = await ds.client.get("/-/telemetry-registry-boom") + assert response.status_code == 500 + finally: + ds.pm.unregister(name="telemetry-registry-boom") + return ds + + +@pytest_asyncio.fixture +async def emitted(otel_spans): + """ + Every (span name, span kind, attributes) triple emitted by exercise(). + The kind is needed to resolve the dynamically named request span. + """ + ds = await exercise() + spans = otel_spans.get_finished_spans() + assert spans, "no spans captured - the fixture is not exercising anything" + # str() so failure messages show plain strings, not registry instances + collected = tuple( + ( + str(span.name), + span.kind, + {str(key): value for key, value in (span.attributes or {}).items()}, + ) + for span in spans + ) + ds.close() + return collected + + +def _partition(emitted): + "The statically named spans, and the dynamically named request spans." + static = [record for record in emitted if record[1] is not SpanKind.SERVER] + server = [record for record in emitted if record[1] is SpanKind.SERVER] + return static, server + + +def _keys_by_span(records): + by_span = {} + for name, _kind, attributes in records: + by_span.setdefault(name, set()).update(attributes) + return by_span + + +@pytest.mark.asyncio +async def test_workload_emits_exactly_the_expected_names(emitted): + "Emitted span and attribute names match the expected literals." + static, server = _partition(emitted) + by_span = _keys_by_span(static) + assert set(by_span) == EXPECTED_SPANS + assert by_span == EXPECTED_ATTRIBUTES + + assert server, "the workload made HTTP requests but no SERVER span was emitted" + union = set() + methods = set() + for name, _kind, attributes in server: + union |= set(attributes) + route = attributes.get("http.route") + # Every request in the workload matches a route + assert route, f"the request span {name!r} carries no http.route" + method, _, name_route = name.partition(" ") + assert name_route == route, ( + f"the request span is named {name!r}, which is not the " + f"`{{method}} {{route}}` of {method!r} and {route!r}" + ) + methods.add(method) + assert methods == EXPECTED_HTTP_METHOD_NAMES + assert union == EXPECTED_HTTP_ATTRIBUTES + + +def test_registry_matches_the_expected_names(): + "Registry names match the expected literals." + assert {str(span) for span in reg.SPANS} == EXPECTED_REGISTRY_NAMES + for span in reg.SPANS: + assert { + str(attribute) for attribute in span.attributes + } == EXPECTED_REGISTRY_ATTRIBUTES[str(span)], f"{span} attributes have drifted" + + +@pytest.mark.asyncio +async def test_every_emitted_span_is_registered(emitted): + "A span added without a registry entry would be missing from the docs." + unregistered = sorted( + {name for name, kind, _ in emitted if reg.span_for(name, kind) is None} + ) + assert ( + not unregistered + ), f"these spans are emitted but not in telemetry_registry.SPANS: {unregistered}" + + +@pytest.mark.asyncio +async def test_every_emitted_attribute_is_registered(emitted): + "An attribute added without a registry entry would be missing from the docs." + unregistered = sorted( + { + f"{name} -> {key}" + for name, kind, keys in emitted + for key in keys + if not reg.attribute_allowed(reg.span_for(name, kind), key) + } + ) + assert ( + not unregistered + ), "these span attributes are emitted but not registered: " + ", ".join( + unregistered + ) + + +@pytest.mark.asyncio +async def test_every_registered_span_is_emitted(emitted): + "The docs should not describe a span that is no longer emitted." + # Compare by identity: the request span's registry name never appears on + # the wire. + resolved = {id(reg.span_for(name, kind)) for name, kind, _ in emitted} + missing = sorted(str(span) for span in reg.SPANS if id(span) not in resolved) + assert not missing, ( + f"these spans are documented but never emitted by the workload: {missing}. " + "Either the instrumentation was removed, or exercise() no longer reaches it." + ) + + +@pytest.mark.asyncio +async def test_every_registered_attribute_is_emitted(emitted): + """ + Every registered attribute, including optional ones, is emitted at least + once. If a new attribute only appears in rare cases, extend exercise(). + """ + by_entry = {} + for name, kind, keys in emitted: + entry = reg.span_for(name, kind) + if entry is not None: + by_entry.setdefault(id(entry), set()).update(keys) + missing = [] + for span in reg.SPANS: + emitted_keys = by_entry.get(id(span), set()) + for attribute in span.attributes: + if attribute not in emitted_keys: + missing.append(f"{span} -> {attribute}") + assert not missing, ( + "these attributes are documented but never emitted by the workload: " + + ", ".join(sorted(missing)) + ) + + +def test_registry_has_no_duplicate_names(): + assert len(set(reg.SPANS)) == len(reg.SPANS) + for span in reg.SPANS: + assert len(set(span.attributes)) == len( + span.attributes + ), f"{span} lists an attribute twice" + + +def test_registry_entries_are_documented(): + "Every entry has a description, used to generate the docs." + for span in reg.SPANS: + assert span.description.strip(), f"{span} has no description" + for attribute in span.attributes: + assert attribute.description.strip(), f"{span} -> {attribute} has none" + + +def test_registry_entries_are_usable_as_plain_strings(): + assert isinstance(reg.DB_QUERY, str) + assert isinstance(reg.DB_NAMESPACE, str) + assert reg.DB_QUERY == "db.query" + assert reg.DB_NAMESPACE == "db.namespace" + assert f"{reg.DB_QUERY}.execute" == "db.query.execute" + + +def test_registry_entries_survive_deepcopy_and_pickle(): + """ + A copied or unpickled entry is a plain str. ConsoleMetricExporter + deepcopies metric attributes, which use registry entries as keys. + """ + for entry in (reg.DB_NAMESPACE, reg.DB_QUERY, reg.M_OPERATION_DURATION): + assert copy.deepcopy({entry: 1}) == {str(entry): 1} + assert type(copy.deepcopy(entry)) is str + assert pickle.loads(pickle.dumps(entry)) == str(entry) + # The original entry keeps its metadata + assert entry.description.strip() + + +@pytest.mark.asyncio +async def test_console_metric_exporter_renders_core_metric_points(otel_metrics): + from opentelemetry.sdk.metrics.export import ( + ConsoleMetricExporter, + MetricExportResult, + ) + + name = _unique("registry_console_export") + ds = Datasette(memory=True) + ds.add_memory_database(name) + await ds.invoke_startup() + # Produces a db.client.operation.duration point keyed by DB_NAMESPACE + await ds.get_database(name).execute("select 1") + + data = otel_metrics.reader.get_metrics_data() + assert data is not None, "no metrics captured - nothing to export" + exporter = ConsoleMetricExporter(out=io.StringIO()) + assert exporter.export(data) is MetricExportResult.SUCCESS + ds.close() + + +def test_every_histogram_declares_bucket_boundaries(): + """ + Every histogram declares bucket boundaries, and only histograms do. + OpenTelemetry's defaults are meant for milliseconds, not seconds. + """ + for metric in reg.METRICS: + if metric.kind == reg.HISTOGRAM: + assert metric.buckets, f"{metric} is a histogram with no boundaries" + assert list(metric.buckets) == sorted( + set(metric.buckets) + ), f"{metric} boundaries must be ascending and unique" + assert metric.buckets[0] > 0, f"{metric} has a non-positive boundary" + else: + assert ( + metric.buckets is None + ), f"{metric} is a {metric.kind} and cannot have bucket boundaries" + + +def test_dynamic_span_lookup(): + """ + dynamic=True entries such as the request span match on kind. They never + match without a kind, and never override a registered name. + """ + assert reg.span_for("GET", SpanKind.SERVER) is reg.HTTP_REQUEST + assert reg.span_for("POST /^/(?P[^/]+)$", SpanKind.SERVER) is ( + reg.HTTP_REQUEST + ) + assert reg.span_for("GET") is None + assert reg.span_for("anything at all", SpanKind.INTERNAL) is None + assert reg.span_for("db.query", SpanKind.SERVER) is reg.DB_QUERY + + +def test_span_and_attribute_lookup(): + assert reg.span_for("db.query") is reg.DB_QUERY + assert reg.span_for("datasette.startup") is reg.STARTUP + assert reg.span_for("not.a.datasette.span") is None + assert reg.attribute_allowed(reg.DB_QUERY, "db.namespace") + assert not reg.attribute_allowed(reg.DB_QUERY, "db.namespace.extra") + assert not reg.attribute_allowed(reg.DB_QUERY, "datasette.isolated_connection") + assert not reg.attribute_allowed(None, "db.namespace") + + +# --- Metric conformance ---------------------------------------------------- + + +@pytest_asyncio.fixture +async def emitted_metrics(otel_metrics): + """ + Metric names and (metric name, attribute key) pairs from a broad workload. + Checks use attribute keys rather than values, since other Datasette + instances in the session can also report points. + """ + # Reaches every synchronous metric except datasette.sql.queries.interrupted + ds = await exercise() + + # datasette.sql.queries.interrupted ignores custom_time_limit timeouts, so + # this needs an instance with a low sql_time_limit_ms. + slow_name = _unique("registry_metrics_slow") + slow = Datasette(memory=True, settings={"sql_time_limit_ms": 5}) + slow.add_memory_database(slow_name) + await slow.invoke_startup() + slow_db = slow.get_database(slow_name) + with pytest.raises(QueryInterrupted): + await slow_db.execute( + "with recursive c(x) as (select 0 union all select x+1 from c) " + "select * from c" + ) + + # Collect before closing the instances so the observable gauges report them + otel_metrics.collect() + snapshot = otel_metrics.snapshot + assert snapshot, "no metrics captured - the fixture is not exercising anything" + pairs = set() + for metric_name, points in snapshot.items(): + for point in points: + for key in point.attributes or {}: + pairs.add((metric_name, key)) + ds.close() + slow.close() + return {"names": set(snapshot), "pairs": pairs, "collector": otel_metrics} + + +@pytest.mark.asyncio +async def test_metrics_conform_to_the_registry(emitted_metrics): + """ + Emitted metric names, kinds, units, attribute keys and enum values match + the registry, using the plugin testing helper. + """ + assert_metrics_conform( + reg.METRICS, emitted_metrics["collector"], scope_name="datasette" + ) + + +@pytest.mark.asyncio +async def test_every_registered_metric_is_emitted(emitted_metrics): + assert_metrics_covered( + reg.METRICS, emitted_metrics["collector"], scope_name="datasette" + ) + + +@pytest.mark.asyncio +async def test_every_registered_metric_attribute_is_emitted(emitted_metrics): + "Every registered metric attribute, including optional ones, is emitted." + emitted_keys_by_metric = {} + for metric_name, key in emitted_metrics["pairs"]: + emitted_keys_by_metric.setdefault(metric_name, set()).add(key) + + missing = [] + for metric in reg.METRICS: + if str(metric) not in emitted_metrics["names"]: + # Reported by test_every_registered_metric_is_emitted + continue + emitted_keys = emitted_keys_by_metric.get(str(metric), set()) + for attribute in metric.attributes: + if attribute not in emitted_keys: + missing.append(f"{metric} -> {attribute}") + assert not missing, ( + "these metric attributes are documented but never emitted by the " + "test workload: " + ", ".join(sorted(missing)) + ) + + +def test_prefix_span_lookup(): + "prefix=True matching, which core does not use but plugin registries can." + hook = reg.SpanName("myplugin.hook.", "A hypothetical span family", prefix=True) + spans = reg.SPANS + (hook,) + assert reg.span_for("myplugin.hook.render_cell", spans=spans) is hook + assert reg.span_for("myplugin.hook.anything", spans=spans) is hook + assert reg.span_for("myplugin.hookish", spans=spans) is None + assert reg.span_for("db.query", spans=spans) is reg.DB_QUERY + + +def test_exact_match_wins_over_prefix(): + family = reg.SpanName("db.", "Greedy prefix", prefix=True) + spans = (family,) + reg.SPANS + assert reg.span_for("db.query", spans=spans) is reg.DB_QUERY + assert reg.span_for("db.anything-else", spans=spans) is family + + +def test_attribute_values_enum_enforced(): + outcome = reg.Attribute("myplugin.outcome", "Enum.", values={"ok", "error"}) + open_attr = reg.Attribute("myplugin.note", "Open value set.") + span = reg.SpanName("myplugin.job", "Test span", (outcome, open_attr)) + assert reg.attribute_value_allowed(span, "myplugin.outcome", "ok") + assert not reg.attribute_value_allowed(span, "myplugin.outcome", "surprise") + assert reg.attribute_value_allowed(span, "myplugin.note", "anything at all") + assert not reg.attribute_value_allowed(span, "not.registered", "x") + assert not reg.attribute_value_allowed(None, "myplugin.outcome", "ok") diff --git a/tests/test_telemetry_testing_kit.py b/tests/test_telemetry_testing_kit.py new file mode 100644 index 00000000..ddf3b3f6 --- /dev/null +++ b/tests/test_telemetry_testing_kit.py @@ -0,0 +1,320 @@ +""" +Tests for datasette.telemetry_testing and the public registry classes, using +a toy plugin registry and instrumentation scope. +""" + +import pytest + +pytest.importorskip("opentelemetry.sdk") + +from opentelemetry import trace as otel_trace + +from datasette import telemetry_registry as reg +from datasette.telemetry import linked_root_span_kwargs +from datasette.telemetry_testing import ( + assert_package_never_imports_sdk, + assert_spans_conform, + assert_spans_covered, +) + +SCOPE = "toyplugin" + +OUTCOME = reg.Attribute( + "toyplugin.outcome", "How the job ended.", values={"ok", "error"} +) +JOB_NAME = reg.Attribute("toyplugin.job", "The job's registered name.") +JOB = reg.SpanName("toyplugin.job.run", "One job execution.", (OUTCOME, JOB_NAME)) +CHAT = reg.SpanName( + "toyplugin.chat ", "One model call, named `toyplugin.chat {model}`.", prefix=True +) +TOY_SPANS = (JOB, CHAT) + +toy_tracer = otel_trace.get_tracer(SCOPE, "0.1") + + +def _toy_spans(otel_spans): + return [ + span + for span in otel_spans.get_finished_spans() + if span.instrumentation_scope and span.instrumentation_scope.name == SCOPE + ] + + +def _run_workload(): + with toy_tracer.start_as_current_span(JOB) as span: + span.set_attribute(OUTCOME, "ok") + span.set_attribute(JOB_NAME, "nightly") + with toy_tracer.start_as_current_span("toyplugin.chat gpt-5"): + pass + + +def test_conformance_passes_for_a_conforming_workload(otel_spans): + _run_workload() + finished = otel_spans.get_finished_spans() + assert_spans_conform(TOY_SPANS, finished, scope_name=SCOPE) + # The chat span matches the CHAT prefix entry: + assert_spans_covered(TOY_SPANS, finished, scope_name=SCOPE) + + +def test_conformance_catches_an_unregistered_span(otel_spans): + with toy_tracer.start_as_current_span("toyplugin.surprise"): + pass + with pytest.raises(AssertionError, match="unregistered span"): + assert_spans_conform( + TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE + ) + + +def test_conformance_catches_an_unregistered_attribute(otel_spans): + with toy_tracer.start_as_current_span(JOB) as span: + span.set_attribute("toyplugin.stealth", 1) + with pytest.raises(AssertionError, match="unregistered attribute"): + assert_spans_conform( + TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE + ) + + +def test_conformance_enforces_declared_enums(otel_spans): + with toy_tracer.start_as_current_span(JOB) as span: + span.set_attribute(OUTCOME, "surprise") + with pytest.raises(AssertionError, match="not in the declared enum"): + assert_spans_conform( + TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE + ) + + +def test_coverage_catches_a_never_emitted_span(otel_spans): + with toy_tracer.start_as_current_span(JOB) as span: + span.set_attribute(OUTCOME, "ok") + span.set_attribute(JOB_NAME, "nightly") + # CHAT never emitted + with pytest.raises(AssertionError, match="never emitted"): + assert_spans_covered( + TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE + ) + + +def test_scope_filter_ignores_other_scopes(otel_spans): + # Spans from other scopes, including Datasette's own, are ignored: + other = otel_trace.get_tracer("someone-else", "1.0") + with other.start_as_current_span("not.in.the.toy.registry"): + pass + _run_workload() + assert_spans_conform(TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE) + + +def test_linked_root_span_kwargs_links_without_parenting(otel_spans): + with toy_tracer.start_as_current_span("toyplugin.cause") as cause: + cause_context = cause.get_span_context() + kwargs = linked_root_span_kwargs() + with toy_tracer.start_as_current_span("toyplugin.effect", **kwargs): + pass + effect = next( + span for span in _toy_spans(otel_spans) if span.name == "toyplugin.effect" + ) + assert effect.parent is None, "must be a root, not a child" + assert effect.context.trace_id != cause_context.trace_id + assert len(effect.links) == 1 + assert effect.links[0].context.span_id == cause_context.span_id + + +def test_linked_root_span_kwargs_with_no_current_span(otel_spans): + kwargs = linked_root_span_kwargs() + assert kwargs["links"] == [] + with toy_tracer.start_as_current_span("toyplugin.orphanless", **kwargs): + pass + span = _toy_spans(otel_spans)[0] + assert span.parent is None + assert span.links == () + + +def test_kit_module_itself_never_imports_the_sdk(): + """ + The kit imports the SDK lazily, so plugins can import it at module level. + + conftest.py runs this test first by name. Update it there if you rename it. + """ + assert_package_never_imports_sdk("datasette.telemetry_testing") + + +# --- Metric conformance helpers -------------------------------------------- + +import itertools + +from opentelemetry import metrics as otel_metrics_api + +from datasette.telemetry_testing import ( + assert_metrics_conform, + assert_metrics_covered, +) + +toy_meter = otel_metrics_api.get_meter(SCOPE, "0.1") + +# Gives each test a unique instrument name: +_metric_ids = itertools.count() + + +def _toy_metric_registry(name, kind="Counter", unit="{job}", attributes=None): + return ( + reg.MetricName( + name, + kind, + unit, + "A toy metric.", + attributes if attributes is not None else (OUTCOME,), + ), + ) + + +def test_metrics_conform_passes_and_covers(otel_metrics): + name = f"toyplugin.jobs.{next(_metric_ids)}" + registry = _toy_metric_registry(name) + counter = toy_meter.create_counter(name, unit="{job}", description="Jobs run") + counter.add(1, {OUTCOME: "ok"}) + otel_metrics.collect() + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + assert_metrics_covered(registry, otel_metrics, scope_name=SCOPE) + + +def test_metrics_conform_catches_unregistered_metric(otel_metrics): + name = f"toyplugin.stealth.{next(_metric_ids)}" + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1) + otel_metrics.collect() + with pytest.raises(AssertionError, match="unregistered metric"): + assert_metrics_conform((), otel_metrics, scope_name=SCOPE) + + +def test_metrics_conform_catches_kind_mismatch(otel_metrics): + name = f"toyplugin.kindclash.{next(_metric_ids)}" + registry = _toy_metric_registry(name, kind="Histogram", unit="{job}") + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {OUTCOME: "ok"}) + otel_metrics.collect() + with pytest.raises(AssertionError, match="registry declares Histogram"): + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +def test_metrics_conform_catches_unit_mismatch(otel_metrics): + name = f"toyplugin.unitclash.{next(_metric_ids)}" + registry = _toy_metric_registry(name, unit="s") + counter = toy_meter.create_counter(name, unit="ms") + counter.add(1, {OUTCOME: "ok"}) + otel_metrics.collect() + with pytest.raises(AssertionError, match="unit"): + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +def test_metrics_conform_catches_unregistered_attribute(otel_metrics): + name = f"toyplugin.attrclash.{next(_metric_ids)}" + registry = _toy_metric_registry(name) + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {"toyplugin.stealth": "x"}) + otel_metrics.collect() + with pytest.raises(AssertionError, match="unregistered attribute"): + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +def test_metrics_conform_enforces_declared_enums(otel_metrics): + name = f"toyplugin.enumclash.{next(_metric_ids)}" + registry = _toy_metric_registry(name) + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {OUTCOME: "surprise"}) + otel_metrics.collect() + with pytest.raises(AssertionError, match="not in the declared enum"): + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +def test_metrics_covered_catches_never_collected(otel_metrics): + registered_but_never_created = _toy_metric_registry( + f"toyplugin.ghost.{next(_metric_ids)}" + ) + otel_metrics.collect() + with pytest.raises(AssertionError, match="never collected"): + assert_metrics_covered( + registered_but_never_created, otel_metrics, scope_name=SCOPE + ) + + +def test_metrics_covered_skips_optional_attributes(otel_metrics): + name = f"toyplugin.optattr.{next(_metric_ids)}" + error_type = reg.Attribute("toyplugin.error", "Only on failure.", optional=True) + registry = _toy_metric_registry(name, attributes=(OUTCOME, error_type)) + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {OUTCOME: "ok"}) # No error attribute + otel_metrics.collect() + assert_metrics_covered(registry, otel_metrics, scope_name=SCOPE) + + +def test_metrics_scope_filter_ignores_other_scopes(otel_metrics): + # Metrics from other scopes, including Datasette's own, are ignored: + name = f"toyplugin.scoped.{next(_metric_ids)}" + registry = _toy_metric_registry(name) + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {OUTCOME: "ok"}) + other_meter = otel_metrics_api.get_meter("someone-else-metrics", "1.0") + stranger = other_meter.create_counter(f"stranger.{next(_metric_ids)}", unit="x") + stranger.add(1) + otel_metrics.collect() + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +# --- UpDownCounter kind + privacy walk -------------------------------------- + +from datasette.telemetry_testing import assert_no_forbidden_values + + +def test_updown_counter_kind_passes(otel_metrics): + name = f"toyplugin.active.{next(_metric_ids)}" + registry = _toy_metric_registry(name, kind=reg.UPDOWN_COUNTER, unit="{turn}") + updown = toy_meter.create_up_down_counter(name, unit="{turn}") + updown.add(1, {OUTCOME: "ok"}) + otel_metrics.collect() + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +def test_counter_registered_as_updown_fails_on_monotonicity(otel_metrics): + name = f"toyplugin.monoclash.{next(_metric_ids)}" + registry = _toy_metric_registry(name, kind=reg.UPDOWN_COUNTER, unit="{job}") + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {OUTCOME: "ok"}) + otel_metrics.collect() + with pytest.raises(AssertionError, match="is_monotonic"): + assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE) + + +def test_forbidden_values_walk_catches_a_leak(otel_spans, otel_metrics): + secret = "sentinel-token-xyzzy" + with toy_tracer.start_as_current_span(JOB) as span: + span.set_attribute(OUTCOME, "ok") + span.set_attribute(JOB_NAME, f"job for {secret}") + with pytest.raises(AssertionError, match="sentinel-token-xyzzy"): + assert_no_forbidden_values( + {secret}, + finished_spans=otel_spans.get_finished_spans(), + scope_name=SCOPE, + ) + + +def test_forbidden_values_walk_passes_a_clean_workload(otel_spans, otel_metrics): + _run_workload() + name = f"toyplugin.clean.{next(_metric_ids)}" + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {OUTCOME: "ok"}) + otel_metrics.collect() + assert_no_forbidden_values( + {"sentinel-token-xyzzy", "alice@example.com", ""}, + finished_spans=otel_spans.get_finished_spans(), + collector=otel_metrics, + scope_name=SCOPE, + ) + + +def test_forbidden_values_walk_checks_metric_attributes(otel_metrics): + secret = "leaky-metric-value" + name = f"toyplugin.leak.{next(_metric_ids)}" + counter = toy_meter.create_counter(name, unit="{job}") + counter.add(1, {"toyplugin.note": secret}) + otel_metrics.collect() + with pytest.raises(AssertionError, match="leaky-metric-value"): + assert_no_forbidden_values({secret}, collector=otel_metrics, scope_name=SCOPE) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 7923d0e7..691c2d64 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field import pytest -from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT +from datasette.app import TEMPLATE_BASE_CONTEXT, Datasette from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database from datasette.template_contexts import PAGES, documented_context_keys @@ -40,17 +40,17 @@ def test_documented_fields(): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert context_field.help, "{}.{} is missing documentation".format( - klass.__name__, context_field.name - ) + assert ( + context_field.help + ), f"{klass.__name__}.{context_field.name} is missing documentation" @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_has_docstring_and_documented_template(klass): - assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) - assert klass.documented_template, "{} is missing a documented_template".format( - klass.__name__ - ) + assert klass.__doc__, f"{klass.__name__} is missing a docstring" + assert ( + klass.documented_template + ), f"{klass.__name__} is missing a documented_template" def test_from_extra_documentation_comes_from_the_extra_class(): @@ -105,7 +105,7 @@ def isolate_extra_template_vars_plugins(): # for the rest of the process. The contract documents plugin-free # Datasette core, so unregister any non-default plugin that adds # template variables via the extra_template_vars hook - from datasette.plugins import pm, DEFAULT_PLUGINS + from datasette.plugins import DEFAULT_PLUGINS, pm hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} removed = [] @@ -182,18 +182,18 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - "Undocumented keys in {} template context: {} - add them to the " - "page's Context class".format(page_name, sorted(undocumented)) + f"Undocumented keys in {page_name} template context: {sorted(undocumented)} - add them to the " + "page's Context class" ) assert not no_longer_present, ( - "Documented keys missing from {} template context: {} - this would " - "break custom templates".format(page_name, sorted(no_longer_present)) + f"Documented keys missing from {page_name} template context: {sorted(no_longer_present)} - this would " + "break custom templates" ) def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): - assert doc, "Base context key {} is missing docs".format(name) + assert doc, f"Base context key {name} is missing docs" def test_template_context_docs_cover_every_documented_key(): @@ -201,15 +201,14 @@ def test_template_context_docs_cover_every_documented_key(): assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() for name in TEMPLATE_BASE_CONTEXT: - assert "``{}``".format(name) in docs, name + assert f"``{name}``" in docs, name for page_name, klass in PAGES.items(): title = "{} page".format(klass.__name__.removesuffix("Context")) assert title in docs, title for context_field in klass.documented_fields(): - assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( - context_field.name, page_name - ) assert ( - "``{}`` - ``{}``".format(context_field.name, context_field.type_name) - in docs - ), "{} type ({} page)".format(context_field.name, page_name) + f"``{context_field.name}``" in docs + ), f"{context_field.name} ({page_name} page)" + assert ( + f"``{context_field.name}`` - ``{context_field.type_name}``" in docs + ), f"{context_field.name} type ({page_name} page)" diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 5c87f577..10021ddf 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -2,11 +2,17 @@ Tests for the register_token_handler plugin hook. """ +import pytest + from datasette.app import Datasette from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.tokens import TokenHandler, TokenRestrictions, SignedTokenHandler -import pytest +from datasette.tokens import ( + SignedTokenHandler, + TokenHandler, + TokenInvalid, + TokenRestrictions, +) @pytest.fixture @@ -66,10 +72,10 @@ async def test_verify_token_unknown_returns_none(datasette): @pytest.mark.asyncio -async def test_verify_token_bad_signature_returns_none(datasette): - """verify_token() should return None for tokens with bad signatures.""" - result = await datasette.verify_token("dstok_tampered_data_here") - assert result is None +async def test_verify_token_bad_signature_raises(datasette): + """verify_token() should raise TokenInvalid for tokens with bad signatures.""" + with pytest.raises(TokenInvalid): + await datasette.verify_token("dstok_tampered_data_here") @pytest.mark.asyncio @@ -334,5 +340,6 @@ async def test_signed_tokens_disabled(): ds = Datasette(settings={"allow_signed_tokens": False}) with pytest.raises(ValueError, match="Signed tokens are not enabled"): await ds.create_token("test_actor", handler="signed") - # verify_token should return None rather than raising - assert await ds.verify_token("dstok_anything") is None + # verify_token should raise TokenInvalid for a dstok_ token + with pytest.raises(TokenInvalid, match="not enabled"): + await ds.verify_token("dstok_anything") diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 9db211d3..21cfa952 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,4 +1,5 @@ import pytest + from .fixtures import make_app_client @@ -75,10 +76,9 @@ async def test_trace_child_tasks_resets_contextvar_on_exception(): from datasette import tracer before = tracer.trace_task_id.get() - with pytest.raises(ValueError): - with tracer.trace_child_tasks(): - assert tracer.trace_task_id.get() is not None - raise ValueError("simulated error") + with pytest.raises(ValueError), tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") # The contextvar must be reset even though the block raised assert tracer.trace_task_id.get() == before diff --git a/tests/test_utils.py b/tests/test_utils.py index 38ebb51e..68d4a504 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,23 +2,26 @@ Tests for various datasette helper functions. """ -from datasette.app import Datasette -from datasette import utils -from datasette.utils.asgi import Request -from datasette.utils.sqlite import ( - sqlite3, - sqlite_hidden_table_names, - sqlite_table_type, - supports_returning, -) import hashlib import json import os import pathlib -import pytest import tempfile from unittest.mock import patch +import pytest + +from datasette import utils +from datasette.app import Datasette +from datasette.utils.asgi import Request +from datasette.utils.sqlite import ( + sqlite3, + sqlite_derived_table_dependencies, + sqlite_hidden_table_names, + sqlite_table_type, + supports_returning, +) + @pytest.mark.parametrize( "path,expected", @@ -132,7 +135,11 @@ def test_path_from_row_pks(row, pks, expected_path): """ {"CategoryID": 1, "Description": "Soft drinks", "Picture": {"$base64": true, "encoded": "FRwCx60F/g=="}} """.strip(), - ) + ), + ( + {"message": b"hello"}, + '{"message": {"$base64": true, "encoded": "aGVsbG8="}}', + ), ], ) def test_custom_json_encoder(obj, expected): @@ -190,7 +197,7 @@ def test_validate_sql_select_good(good_sql): @pytest.mark.parametrize("open_quote,close_quote", [('"', '"'), ("[", "]")]) def test_detect_fts(open_quote, close_quote): - sql = """ + sql = f""" CREATE TABLE "Dumb_Table" ( "TreeID" INTEGER, "qSpecies" TEXT @@ -205,9 +212,9 @@ def test_detect_fts(open_quote, close_quote): "qCaretaker" TEXT ); CREATE VIEW Test_View AS SELECT * FROM Dumb_Table; - CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open}Street_Tree_List{close}); + CREATE VIRTUAL TABLE {open_quote}Street_Tree_List_fts{close_quote} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open_quote}Street_Tree_List{close_quote}); CREATE VIRTUAL TABLE r USING rtree(a, b, c); - """.format(open=open_quote, close=close_quote) + """ conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) assert None is utils.detect_fts(conn, "Dumb_Table") @@ -221,6 +228,8 @@ def test_detect_fts(open_quote, close_quote): "identifier,expected", ( ("plain", "plain"), + ("plain\n", '"plain\n"'), + ("select\n", '"select\n"'), ("select", '"select"'), ("has space", '"has space"'), ("has'quote", '"has\'quote"'), @@ -258,8 +267,8 @@ def test_escape_sqlite_prevents_injection(): conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") malicious = "users] UNION SELECT password FROM users--" - conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) - sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) + conn.execute(f'CREATE TABLE "{malicious}" (id INTEGER)') + sql = f"select count(*) from {utils.escape_sqlite(malicious)}" results = conn.execute(sql).fetchall() conn.close() # The injected UNION must not execute - only the empty malicious table @@ -269,16 +278,16 @@ def test_escape_sqlite_prevents_injection(): @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): - sql = """ + sql = f""" CREATE TABLE [{table}] ( "TreeID" INTEGER, "qSpecies" TEXT ); CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 ("qSpecies", content="{table}"); - """.format(table=table) + """ conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) - assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table) + assert f"{table}_fts" == utils.detect_fts(conn, table) conn.close() @@ -363,6 +372,46 @@ def test_sqlite_hidden_table_names_hides_multiline_content_fts_table(): conn.close() +def test_sqlite_derived_table_dependencies(): + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table docs(id integer primary key, body text); + create virtual table external_fts5 using fts5( + body, content='docs', content_rowid='id' + ); + create virtual table internal_fts5 using fts5(body); + create virtual table contentless_fts5 using fts5(body, content=''); + create virtual table external_fts4 using fts4(body, content="docs"); + create virtual table internal_fts4 using fts4(body); + create virtual table contentless_fts4 using fts4(body, content=""); + create table [docs, archive](body text); + create virtual table commented_fts5 using fts5( + body, tokenize='porter unicode61', + /* Comments and commas in quoted values must not confuse parsing. */ + content='docs, archive' + ); + create virtual table boxes using rtree(id, minx, maxx, miny, maxy); + """) + + dependencies = sqlite_derived_table_dependencies(conn) + + assert dependencies["external_fts5"] == "docs" + assert dependencies["external_fts4"] == "docs" + assert dependencies["commented_fts5"] == "docs, archive" + assert "contentless_fts5" not in dependencies + assert "contentless_fts4" not in dependencies + assert dependencies["internal_fts5_content"] == "internal_fts5" + assert dependencies["internal_fts4_content"] == "internal_fts4" + assert dependencies["external_fts5_data"] == "external_fts5" + assert dependencies["external_fts4_segments"] == "external_fts4" + assert dependencies["boxes_node"] == "boxes" + assert dependencies["boxes_parent"] == "boxes" + assert dependencies["boxes_rowid"] == "boxes" + finally: + conn.close() + + @pytest.mark.parametrize( "url,expected", [ @@ -686,7 +735,6 @@ def test_resolve_env_secrets(config, expected): [ ({"id": "blah"}, "blah"), ({"id": "blah", "login": "l"}, "l"), - ({"id": "blah", "login": "l"}, "l"), ({"id": "blah", "login": "l", "username": "u"}, "u"), ({"login": "l", "name": "n"}, "n"), ( @@ -752,6 +800,21 @@ def test_parse_metadata(content, expected): ("select 1 + :one + :two", ["one", "two"]), ("select 'bob' || '0:00' || :cat", ["cat"]), ("select this is invalid :one, :two, :three", ["one", "two", "three"]), + # A string literal containing a comment marker should not hide + # parameters that come after it + ("select * from t where note = '-- TODO' and id = :id", ["id"]), + ("select '--' || :y", ["y"]), + ("select * from t where note = '/* x */' and id = :id", ["id"]), + # Parameters that live inside a comment should be ignored + ("select :x -- and :ignored", ["x"]), + ("select :x /* and :ignored */ from t", ["x"]), + ("select :x /* and :ignored", ["x"]), + # Parameters inside quoted identifiers should be ignored + ("select [a:b] from t where id = :id", ["id"]), + ("select `a:b` from t where id = :id", ["id"]), + ("select `a``:b` from t where id = :id", ["id"]), + # Parameters inside a string literal should be ignored + ("select ':ignored' || :real", ["real"]), ), ) @pytest.mark.parametrize("use_async_version", (False, True)) diff --git a/tests/test_utils_check_callable.py b/tests/test_utils_check_callable.py index 4f72f9ff..857b73cd 100644 --- a/tests/test_utils_check_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,6 +1,7 @@ -from datasette.utils.check_callable import check_callable import pytest +from datasette.utils.check_callable import check_callable + class AsyncClass: async def __call__(self): diff --git a/tests/test_utils_permissions.py b/tests/test_utils_permissions.py index bc3599c2..918dab95 100644 --- a/tests/test_utils_permissions.py +++ b/tests/test_utils_permissions.py @@ -1,14 +1,17 @@ +from collections.abc import Callable + import pytest + from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.utils.permissions import resolve_permissions_from_catalog -from typing import Callable, List @pytest.fixture def db(): ds = Datasette() import tempfile + from datasette.database import Database path = tempfile.mktemp(suffix="demo.db") @@ -127,7 +130,7 @@ def plugin_root_deny_for_all() -> Callable[[str], PermissionSQL]: def plugin_conflicting_same_child_rules( user: str, parent: str, child: str -) -> List[Callable[[str], PermissionSQL]]: +) -> list[Callable[[str], PermissionSQL]]: def allow_provider(action: str) -> PermissionSQL: return PermissionSQL( """ @@ -277,9 +280,7 @@ async def test_alice_global_allow_with_specific_denies_catalog(db): # Alice can see everything except accounting/sales and hr/* assert "/accounting/sales" in res_denied(rows) for r in rows: - if r["parent"] == "hr": - assert r["allow"] == 0 - elif r["resource"] == "/accounting/sales": + if r["parent"] == "hr" or r["resource"] == "/accounting/sales": assert r["allow"] == 0 else: assert r["allow"] == 1 diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 979ff9e1..363814d7 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -1,7 +1,7 @@ import pytest -from datasette.utils.sqlite import sqlite3 from datasette.utils.sql_analysis import analyze_sql_tables +from datasette.utils.sqlite import sqlite3 @pytest.fixture @@ -439,7 +439,7 @@ def test_analyze_attached_database_tables(conn): } -def test_analyze_clears_authorizer_on_error(): +def test_analyze_disables_authorizer_on_error(): class FakeConnection: def __init__(self): self.authorizers = [] @@ -455,4 +455,5 @@ def test_analyze_clears_authorizer_on_error(): with pytest.raises(sqlite3.OperationalError): analyze_sql_tables(conn, "bad SQL") - assert conn.authorizers[-1] is None + final_authorizer = conn.authorizers[-1] + assert final_authorizer is None or final_authorizer() == sqlite3.SQLITE_OK diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 88ce5520..45eea483 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -3,14 +3,16 @@ Tests for the write_wrapper plugin hook. """ import asyncio +import sqlite3 +import time from dataclasses import dataclass + +import pytest + from datasette.app import Datasette from datasette.events import Event from datasette.hookspecs import hookimpl from datasette.plugins import pm -import pytest -import sqlite3 -import time @dataclass @@ -113,7 +115,8 @@ async def test_write_wrapper_exception_thrown_into_generator(datasette): def wrapper(conn): try: yield - except Exception as e: + except Exception as e: # noqa: BLE001 + # Test helper deliberately captures whatever the wrapped write raised caught["error"] = e return wrapper @@ -232,7 +235,6 @@ async def test_write_wrapper_return_none_skips(datasette): @hookimpl def write_wrapper(datasette, database, request, transaction): log.append("hook-called") - return None pm.register(Plugin(), name="test_skip") try: @@ -339,7 +341,7 @@ async def test_write_wrapper_via_api(tmp_path): "/test/api_test/-/insert", json={"row": {"name": "test"}, "return": True}, headers={ - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": "application/json", }, ) @@ -349,6 +351,73 @@ async def test_write_wrapper_via_api(tmp_path): pm.unregister(name="test_api") +@pytest.mark.asyncio +@pytest.mark.parametrize("num_sql_threads", (0, 1)) +@pytest.mark.parametrize("in_memory", (True, False)) +@pytest.mark.parametrize( + "operations", + ( + [{"op": "add_column", "args": {"name": "extra", "type": "text"}}], + [{"op": "rename_column", "args": {"name": "id", "to": "renamed_id"}}], + [{"op": "rename_table", "args": {"to": "renamed_t"}}], + [ + {"op": "add_column", "args": {"name": "extra", "type": "text"}}, + {"op": "rename_column", "args": {"name": "id", "to": "renamed_id"}}, + {"op": "rename_table", "args": {"to": "renamed_t"}}, + ], + ), + ids=["add-column", "transform", "rename-table", "combined"], +) +async def test_write_wrapper_can_reject_alter_table_after_write( + tmp_path, num_sql_threads, in_memory, operations +): + """Raising after yield should roll back the schema change.""" + db_path = str(tmp_path / "demo.db") + ds = Datasette( + [] if in_memory else [db_path], + config={"permissions": {"alter-table": True}}, + settings={"num_sql_threads": num_sql_threads}, + ) + db = ( + ds.add_memory_database(db_path, name="demo") + if in_memory + else ds.get_database("demo") + ) + await db.execute_write("CREATE TABLE t (id)") + await db.execute_write("INSERT INTO t (id) VALUES (1)") + before = await db.execute_fn(lambda conn: list(conn.iterdump())) + + class Plugin: + __name__ = "Plugin" + + @staticmethod + @hookimpl + def write_wrapper(database): + def wrapper(conn): + yield + raise ValueError("Rejected after write") + + return wrapper if database == "demo" else None + + pm.register(Plugin(), name="test_reject_alter_table") + try: + response = await ds.client.post( + "/demo/t/-/alter", + json={"operations": operations}, + ) + assert response.status_code == 400 + assert response.json()["errors"] == ["Rejected after write"] + assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before + assert not [ + event + for event in getattr(ds, "_tracked_events", []) + if event.name in ("alter-table", "rename-table") + ] + finally: + pm.unregister(name="test_reject_alter_table") + ds.close() + + @pytest.mark.asyncio async def test_write_wrapper_change_group_pattern(datasette): """Test the motivating use case: activating a change group around a write.""" @@ -466,7 +535,7 @@ async def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny try: request = FakeRequest(actor) if should_deny: - with pytest.raises(Exception): + with pytest.raises(sqlite3.DatabaseError, match="not authorized"): await db.execute_write_fn( lambda conn: conn.execute( f"insert into {table} (value) values ('test')"
      {}{i}{i}{i}a{i}' + f'Ada 17