diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml new file mode 100644 index 00000000..fdbc71c9 --- /dev/null +++ b/.github/actions/setup-sqlite-version/action.yml @@ -0,0 +1,39 @@ +name: "Setup SQLite version" +description: "Build and activate a specific SQLite version from its amalgamation archive" +inputs: + version: + description: "The SQLite version to install" + required: true + cflags: + description: "CFLAGS to use when compiling SQLite" + required: false + default: "" + skip-activate: + description: "Set to true to skip modifying the library path" + required: false + default: "false" + fallback-urls: + description: "Whitespace-separated fallback download URLs to try after sqlite.org" + required: false + default: "" +outputs: + sqlite-location: + description: "Directory containing the compiled SQLite library" + value: ${{ steps.build.outputs.sqlite-location }} +runs: + using: "composite" + steps: + - shell: bash + run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/sqlite-versions/downloads + key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 + - id: build + shell: bash + run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" + env: + SQLITE_VERSION: ${{ inputs.version }} + SQLITE_CFLAGS: ${{ inputs.cflags }} + SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} + SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh new file mode 100644 index 00000000..03d6a68f --- /dev/null +++ b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" +cflags="${SQLITE_CFLAGS:-}" +skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" +extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" + +case "$version_spec" in + 3.46 | 3.46.0) + sqlite_version="3.46.0" + sqlite_year="2024" + amalgamation_id="3460000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" + ;; + 3.25 | 3.25.0) + sqlite_version="3.25.0" + sqlite_year="2018" + amalgamation_id="3250000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1" + ;; + *) + echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." + exit 1 + ;; +esac + +case "$(uname -s)" in + Linux) + library_name="libsqlite3.so.0" + library_path_var="LD_LIBRARY_PATH" + ;; + Darwin) + library_name="libsqlite3.dylib" + library_path_var="DYLD_LIBRARY_PATH" + ;; + *) + echo "::error::Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +runner_temp="${RUNNER_TEMP:-}" +if [ -z "$runner_temp" ]; then + runner_temp="$(mktemp -d)" +fi + +filename="sqlite-amalgamation-${amalgamation_id}" +official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" +download_dir="${runner_temp}/sqlite-versions/downloads" +source_root="${runner_temp}/sqlite-versions/source" +source_dir="${source_root}/${filename}" +build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" +archive_path="${download_dir}/${filename}.zip" + +mkdir -p "$download_dir" "$source_root" "$build_dir" + +download_archive() { + local url + local candidate_path="${archive_path}.tmp" + local urls=("$official_url") + + for url in $builtin_fallback_urls $extra_fallback_urls; do + urls+=("$url") + done + + rm -f "$candidate_path" + for url in "${urls[@]}"; do + echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" + if curl \ + --fail \ + --location \ + --show-error \ + --retry 5 \ + --retry-delay 2 \ + --retry-max-time 180 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 240 \ + --output "$candidate_path" \ + "$url"; then + mv "$candidate_path" "$archive_path" + return 0 + fi + + echo "::warning::Download failed from ${url}" + rm -f "$candidate_path" + done + + echo "::error::Could not download SQLite ${sqlite_version} amalgamation" + return 1 +} + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + if [ ! -f "$archive_path" ]; then + download_archive + fi + + rm -rf "$source_dir" + unzip -q "$archive_path" -d "$source_root" +fi + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" + exit 1 +fi + +read -r -a cflag_args <<< "$cflags" + +echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" +gcc \ + -fPIC \ + -shared \ + "${cflag_args[@]}" \ + "${source_dir}/sqlite3.c" \ + "-I${source_dir}" \ + -o "${build_dir}/${library_name}" + +if [ "$library_name" = "libsqlite3.so.0" ]; then + ln -sf "$library_name" "${build_dir}/libsqlite3.so" +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" +else + echo "sqlite-location=${build_dir}" +fi + +case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in + true | 1 | yes) + echo "Skipping ${library_path_var} activation" + ;; + *) + existing_value="${!library_path_var:-}" + if [ -n "${GITHUB_ENV:-}" ]; then + if [ -n "$existing_value" ]; then + echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" + else + echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" + fi + fi + echo "Added ${build_dir} to ${library_path_var}" + ;; +esac diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml deleted file mode 100644 index e56d9c27..00000000 --- a/.github/workflows/deploy-branch-preview.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy a Datasette branch preview to Vercel - -on: - workflow_dispatch: - inputs: - branch: - description: "Branch to deploy" - required: true - type: string - -jobs: - deploy-branch-preview: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install datasette-publish-vercel - - name: Deploy the preview - env: - VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} - run: | - export BRANCH="${{ github.event.inputs.branch }}" - wget https://latest.datasette.io/fixtures.db - datasette publish vercel fixtures.db \ - --branch $BRANCH \ - --project "datasette-preview-$BRANCH" \ - --token $VERCEL_TOKEN \ - --scope datasette \ - --about "Preview of $BRANCH" \ - --about_url "https://github.com/simonw/datasette/tree/$BRANCH" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 10cdac01..3fc83438 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -1,10 +1,11 @@ name: Deploy latest.datasette.io on: + workflow_dispatch: push: branches: - main - - 1.0-dev + # - 1.0-dev permissions: contents: read @@ -14,24 +15,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 - # Using Python 3.10 for gcloud compatibility: with: - python-version: "3.10" - - uses: actions/cache@v4 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.13" + cache: pip - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] - python -m pip install -e .[docs] + python -m pip install . --group dev python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests if: ${{ github.ref == 'refs/heads/main' }} @@ -64,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py < metadata.json # cat metadata.json - - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@v0 + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 with: - version: '318.0.0' - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Deploy to Cloud Run env: LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} @@ -122,7 +117,7 @@ jobs: --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ + --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \ --install 'datasette-ephemeral-tables>=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index a54bd83a..00000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Read the Docs Pull Request Preview -on: - pull_request_target: - types: - - opened - -permissions: - pull-requests: write - -jobs: - documentation-links: - runs-on: ubuntu-latest - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "datasette" diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000..f5b8dbf6 --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,48 @@ +name: Playwright + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + browser: [chromium, firefox, webkit] + steps: + - uses: actions/checkout@v7 + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: "3.14" + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml + - name: Cache uv + uses: actions/cache@v6 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py3.14-uv- + - name: Cache Playwright browsers + uses: actions/cache@v6 + with: + path: ~/.cache/ms-playwright/ + key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-playwright-${{ matrix.browser }}- + - name: Install uv + run: python -m pip install uv + - name: Install dependencies + run: uv sync --group dev --group playwright + - name: Install ${{ matrix.browser }} + run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }} + - name: Run Playwright tests + run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 77cce7d1..d92ab82b 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 - - uses: actions/cache@v4 + uses: actions/checkout@v7 + - uses: actions/cache@v6 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5acb4899..21ed4c12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,16 +14,16 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev - name: Run tests run: | pytest @@ -35,13 +35,13 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.13' cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install setuptools wheel build @@ -56,16 +56,16 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.10' cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - python -m pip install -e .[docs] + python -m pip install . --group dev python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- @@ -73,12 +73,13 @@ jobs: DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@v0 + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 with: - version: '318.0.0' - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Deploy stable-docs.datasette.io to Cloud Run run: |- gcloud config set run/region us-central1 @@ -91,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index afe8d6b2..c5a4f0db 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 8a47fd2d..58635025 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,16 +9,16 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.11' cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: '**/pyproject.toml' - name: Install dependencies run: | - pip install -e '.[docs]' + pip install . --group dev - name: Check spelling run: | codespell README.md --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml new file mode 100644 index 00000000..ecde5940 --- /dev/null +++ b/.github/workflows/stable-docs.yml @@ -0,0 +1,76 @@ +name: Update Stable Docs + +on: + release: + types: [published] + push: + branches: + - main + +permissions: + contents: write + +jobs: + update_stable_docs: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 # We need all commits to find docs/ changes + - name: Set up Git user + run: | + git config user.name "Automated" + git config user.email "actions@users.noreply.github.com" + - name: Create stable branch if it does not yet exist + run: | + if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then + # Make sure we have all tags locally + git fetch --tags --quiet + + # Latest tag that is just numbers and dots (optionally prefixed with 'v') + # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. + LATEST_RELEASE=$( + git tag -l --sort=-v:refname \ + | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ + | head -n1 + ) + + git checkout -b stable + + # If there are any stable releases, copy docs/ from the most recent + if [ -n "$LATEST_RELEASE" ]; then + rm -rf docs/ + git checkout "$LATEST_RELEASE" -- docs/ || true + fi + + git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" + git push -u origin stable + fi + - name: Handle Release + if: github.event_name == 'release' && !github.event.release.prerelease + run: | + git fetch --all + git checkout stable + git reset --hard ${GITHUB_REF#refs/tags/} + git push origin stable --force + - name: Handle Commit to Main + if: contains(github.event.head_commit.message, '!stable-docs') + run: | + git fetch origin + git checkout -b stable origin/stable + # Get the list of modified files in docs/ from the current commit + FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) + # Check if the list of files is non-empty + if [[ -n "$FILES" ]]; then + # Checkout those files to the stable branch to over-write with their contents + for FILE in $FILES; do + git checkout ${{ github.sha }} -- $FILE + done + git add docs/ + git commit -m "Doc changes from ${{ github.sha }}" + git push origin stable + else + echo "No changes to docs/ in this commit." + exit 0 + fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 22a69150..e9bd4bab 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,17 +15,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.12' cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: '**/pyproject.toml' - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] + python -m pip install . --group dev python -m pip install pytest-cov - name: Run tests run: |- diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 7357b30c..5e81ed82 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,15 +12,15 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: python-version: "3.10" cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 698aec8a..2fdb3a40 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,16 +25,16 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + uses: ./.github/actions/setup-sqlite-version with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" @@ -45,7 +45,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev pip freeze - name: Run tests run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 901c4905..751eedfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,23 +9,24 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev pip freeze - name: Run tests run: | @@ -33,9 +34,12 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Install docs dependencies + - name: Black run: | - pip install -e '.[docs]' + black --version + black --check . + - name: Ruff + run: ruff check datasette tests - name: Check if cog needs to be run run: | cog --check docs/*.rst diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index fcee0f21..f2c074a6 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 9792245d..5b8818c3 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -5,11 +5,14 @@ on: permissions: contents: read + models: read jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 277ff653..2a7f6620 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,20 @@ build-metadata.json datasets.json +.playwright-mcp + scratchpad +ignored/ + .vscode +uv.lock +data.db + +# test databases +*.db + # We don't use Pipfile, so ignore them Pipfile Pipfile.lock @@ -123,4 +133,6 @@ node_modules # include it in source control. tests/*.dylib tests/*.so -tests/*.dll \ No newline at end of file +tests/*.dll + +.idea diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 5b30e75a..8b3e54aa 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,16 +1,17 @@ version: 2 -build: - os: ubuntu-20.04 - tools: - python: "3.11" - sphinx: - configuration: docs/conf.py + configuration: docs/conf.py -python: - install: - - method: pip - path: . - extra_requirements: - - docs +build: + os: ubuntu-24.04 + tools: + python: "3.13" + jobs: + install: + - pip install --upgrade pip + - pip install . --group dev + +formats: +- pdf +- epub diff --git a/Justfile b/Justfile index 172de444..6ffff870 100644 --- a/Justfile +++ b/Justfile @@ -5,38 +5,73 @@ export DATASETTE_SECRET := "not_a_secret" # Setup project @init: - pipenv run pip install -e '.[test,docs]' + uv sync # Run pytest with supplied options -@test *options: - pipenv run pytest {{options}} +@test *options: init + uv run pytest -n auto {{options}} + +# Install Playwright browser support, Chromium by default +@playwright-install browser="chromium": + uv run --group playwright playwright install {{browser}} + +# Install all Playwright browsers used by the test suite +@playwright-install-all: + uv run --group playwright playwright install chromium firefox webkit + +# Run Playwright tests, Chromium by default +@playwright browser="chromium" *options: + uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}} + +# Run Playwright tests against all supported browsers +@playwright-all *options: + uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}} @codespell: - pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt - pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - pipenv run codespell tests --ignore-words docs/codespell-ignore-words.txt + uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + uv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, flake8, mypy, cog +# Run linters: black, ruff, prettier, cog @lint: codespell - pipenv run black . --check - pipenv run flake8 - pipenv run cog --check README.md docs/*.rst + uv run black datasette tests --check + uv run ruff check datasette tests + npm run prettier -- --check + uv run cog --check README.md docs/*.rst + +# Apply ruff fixes +@fix: + uv run ruff check --fix datasette tests # Rebuild docs with cog @cog: - pipenv run cog -r README.md docs/*.rst + uv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: cog - pipenv run blacken-docs -l 60 docs/*.rst - cd docs && pipenv run make livehtml +@docs: cog blacken-docs + uv run make -C docs livehtml + +# Build docs as static HTML +@docs-build: cog blacken-docs + rm -rf docs/_build && cd docs && uv run make html # Apply Black @black: - pipenv run black . + uv run black datasette tests -@serve: - pipenv run sqlite-utils create-database data.db - pipenv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore - pipenv run python -m datasette data.db --root --reload +# Apply blacken-docs +@blacken-docs: + uv run blacken-docs -l 60 docs/*.rst + +# Apply prettier +@prettier: + npm run fix + +# Format code with both black and prettier +@format: black prettier blacken-docs + +@serve *options: + uv run sqlite-utils create-database data.db + uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + uv run python -m datasette data.db --root --reload {{options}} diff --git a/datasette/__init__.py b/datasette/__init__.py index 47d2b4f6..e0022178 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,7 +1,14 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa +from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa +from datasette.utils.asgi import ( # noqa + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py new file mode 100644 index 00000000..103c616d --- /dev/null +++ b/datasette/_pytest_plugin.py @@ -0,0 +1,123 @@ +""" +Pytest plugin that automatically closes any Datasette instances constructed +during a pytest test — both in the test body and in function-scoped +fixtures. Instances constructed by session-, module-, class- or package- +scoped fixtures are left alone, because other tests in the session will +still want to use them. + +Registered as a pytest11 entry point in pyproject.toml so that downstream +projects using Datasette get the same FD-safety net for their own tests. + +Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the +equivalent ini file). +""" + +from __future__ import annotations + +import contextvars +import weakref + +import pytest + +_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "datasette_active_instances", default=None +) + +_original_init = None + + +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init + + +def pytest_configure(config): + if _enabled(config): + _install_tracking() + + +def pytest_addoption(parser): + parser.addini( + "datasette_autoclose", + help=( + "Automatically close Datasette instances created inside test " + "bodies and function-scoped fixtures (default: true)." + ), + default="true", + ) + + +def _enabled(config) -> bool: + value = config.getini("datasette_autoclose") + if isinstance(value, bool): + return value + return str(value).strip().lower() not in ("false", "0", "no", "off") + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + """Track Datasette instances across setup, call and teardown; close at end.""" + if not _enabled(item.config): + yield + return + refs: list[weakref.ref] = [] + token = _active_instances.set(refs) + try: + yield + finally: + _active_instances.reset(token) + for ref in reversed(refs): + ds = ref() + if ds is None: + continue + try: + ds.close() + except Exception as e: + item.warn( + pytest.PytestUnraisableExceptionWarning( + f"Error closing Datasette instance: {e!r}" + ) + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request): + """Exempt instances created by non-function-scoped fixtures. + + Session-, module-, class- and package-scoped fixtures produce Datasette + instances that must survive beyond the current test — other tests in + the session will still use them. When such a fixture creates one or + more Datasette instances during its setup, we snapshot the tracking + list before the fixture runs and subtract off any instances that were + added during its setup, so they don't get closed at test teardown. + """ + refs = _active_instances.get() + if refs is None: + yield + return + before_ids = {id(ref) for ref in refs} + yield + if fixturedef.scope != "function": + new_refs = [ref for ref in refs if id(ref) not in before_ids] + for new_ref in new_refs: + try: + refs.remove(new_ref) + except ValueError: + pass diff --git a/datasette/app.py b/datasette/app.py index 6c7026a8..0e31273d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,13 +1,17 @@ -from asgi_csrf import Errors +from __future__ import annotations + import asyncio -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union -import asgi_csrf +import contextvars +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence + +if TYPE_CHECKING: + from datasette.permissions import Resource + from datasette.tokens import TokenRestrictions import collections import dataclasses import datetime import functools import glob -import hashlib import httpx import importlib.metadata import inspect @@ -30,18 +34,44 @@ from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, + pass_context, PrefixLoader, ) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .events import Event +from .column_types import SQLiteType +from . import stored_queries, write_sql from .views import Context -from .views.database import database_download, DatabaseView, TableCreateView, QueryView +from .views.database import ( + database_download, + DatabaseView, + QueryView, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + QueryEditView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) from .views.index import IndexView from .views.special import ( JsonDataView, PatternPortfolioView, + AutocompleteDebugView, AuthTokenView, ApiExplorerView, CreateTokenView, @@ -52,11 +82,18 @@ from .views.special import ( AllowedResourcesView, PermissionRulesView, PermissionCheckView, + JumpView, + InstanceSchemaView, + DatabaseSchemaView, + TableSchemaView, ) from .views.table import ( + TableAutocompleteView, TableInsertView, TableUpsertView, + TableSetColumnTypeView, TableDropView, + TableFragmentView, table_view, ) from .views.row import RowView, RowDeleteView, RowUpdateView @@ -65,6 +102,7 @@ from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( + PaginatedResources, PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, @@ -73,6 +111,7 @@ from .utils import ( baseconv, call_with_supported_arguments, detect_json1, + add_cors_headers, display_actor, escape_css_string, escape_sqlite, @@ -84,14 +123,18 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, + sha256_file, tilde_decode, + tilde_encode, to_css_class, urlsafe_components, redact_keys, row_sql_params_pks, ) +from .tokens import TokenInvalid from .utils.asgi import ( AsgiLifespan, + BadRequest, Forbidden, NotFound, DatabaseNotFound, @@ -105,6 +148,7 @@ from .utils.asgi import ( asgi_send_file, asgi_send_redirect, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -114,10 +158,39 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .utils.permissions import build_rules_union, PluginSQL +from .resources import DatabaseResource, TableResource app_root = Path(__file__).parent.parent + +# Context variable to track when code is executing within a datasette.client request +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + + +class _DatasetteClientContext: + """Context manager to mark code as executing within a datasette.client request.""" + + def __enter__(self): + self.token = _in_datasette_client.set(True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + _in_datasette_client.reset(self.token) + return False + + +@dataclasses.dataclass +class PermissionCheck: + """Represents a logged permission check for debugging purposes.""" + + when: str + actor: Dict[str, Any] | None + action: str + parent: str | None + child: str | None + result: bool + + # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 SQLITE_LIMIT_ATTACHED = 10 @@ -136,6 +209,11 @@ SETTINGS = ( 100, "Maximum rows that can be inserted at a time using the bulk insert API", ), + Setting( + "max_post_body_bytes", + 2 * 1024 * 1024, + "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit", + ), Setting( "num_sql_threads", 3, @@ -227,12 +305,24 @@ FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() +ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) + + +def _permission_cache_key(actor, action, parent, child): + # Key on the full serialized actor so actors differing in any field + # (e.g. token restrictions) never share cache entries + actor_key = ( + json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None + ) + return (actor_key, action, parent, child) + + async def favicon(request, send): await asgi_send_file( send, str(FAVICON_PATH), content_type="image/png", - headers={"Cache-Control": "max-age=3600, immutable, public"}, + headers={"Cache-Control": "max-age=3600, public"}, ) @@ -249,6 +339,57 @@ def _to_string(value): return json.dumps(value, default=str) +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + +def _resolve_static_asset_path(root_path, path): + root = Path(root_path).resolve() + full_path = (root / path).resolve() + try: + full_path.relative_to(root) + except ValueError: + raise ValueError("Static asset path cannot escape static root") from None + return full_path + + +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", + "show_logout": "True if the logout link should be shown in the navigation menu", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` diff --git a/datasette/templates/_codemirror.html b/datasette/templates/_codemirror.html index c4629aeb..75c16168 100644 --- a/datasette/templates/_codemirror.html +++ b/datasette/templates/_codemirror.html @@ -1,5 +1,5 @@ - - + + diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html index 034e9678..570bb37e 100644 --- a/datasette/templates/_facet_results.html +++ b/datasette/templates/_facet_results.html @@ -12,9 +12,9 @@ + {% if queries_more %} +

View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

+ {% endif %} {% endif %} {% if tables %} -

Tables

+

Tables schema

{% endif %} {% for table in tables %} @@ -64,7 +76,7 @@

{{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

-

{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

+

{% if table.count is none %}Many rows{% elif table.count_truncated %}>{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

{% endif %} {% endfor %} @@ -87,5 +99,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/debug_actions.html b/datasette/templates/debug_actions.html new file mode 100644 index 00000000..c9dccaaa --- /dev/null +++ b/datasette/templates/debug_actions.html @@ -0,0 +1,43 @@ +{% extends "base.html" %} + +{% block title %}Registered Actions{% endblock %} + +{% block content %} +

Registered actions

+ +{% set current_tab = "actions" %} +{% include "_permissions_debug_tabs.html" %} + +

+ This Datasette instance has registered {{ data.actions|length }} action{{ data.actions|length != 1 and "s" or "" }}. + Actions are used by the permission system to control access to different features. +

+ + + + + + + + + + + + + + + {% for action in data.actions %} + + + + + + + + + + {% endfor %} + +
NameAbbrDescriptionResourceTakes ParentTakes ChildAlso Requires
{{ action.name }}{% if action.abbr %}{{ action.abbr }}{% endif %}{{ action.description or "" }}{% if action.resource_class %}{{ action.resource_class }}{% endif %}{% if action.takes_parent %}✓{% endif %}{% if action.takes_child %}✓{% endif %}{% if action.also_requires %}{{ action.also_requires }}{% endif %}
+ +{% endblock %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 031ff07d..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,14 +3,16 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} - + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} {% block content %} +

Allowed resources

-

Allowed Resources

+{% set current_tab = "allowed" %} +{% include "_permissions_debug_tabs.html" %}

Use this tool to check which resources the current actor is allowed to access for a given permission action. It queries the /-/allowed.json API endpoint.

@@ -21,13 +23,13 @@ {% endif %}
- +
Only certain actions are supported by this endpoint @@ -42,12 +44,12 @@
- Filter results to a specific child resource (requires parent) + Filter results to a specific child resource (requires parent to be set)
- + Number of results per page (max 200)
@@ -80,35 +82,19 @@ const resultsContent = document.getElementById('results-content'); const resultsCount = document.getElementById('results-count'); const pagination = document.getElementById('pagination'); const submitBtn = document.getElementById('submit-btn'); -let currentData = null; - -form.addEventListener('submit', async (ev) => { - ev.preventDefault(); - updateURL('allowed-form', 1); - await fetchResults(1, false); -}); - -// Handle browser back/forward -window.addEventListener('popstate', () => { - const params = populateFormFromURL(); - const action = params.get('action'); - const page = params.get('page'); - if (action) { - fetchResults(page ? parseInt(page) : 1, false); - } -}); +const hasDebugPermission = {{ 'true' if has_debug_permission else 'false' }}; // Populate form on initial load (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('page'); + const page = params.get('_page'); if (action) { - fetchResults(page ? parseInt(page) : 1, false); + fetchResults(page ? parseInt(page) : 1); } })(); -async function fetchResults(page = 1, updateHistory = true) { +async function fetchResults(page = 1) { submitBtn.disabled = true; submitBtn.textContent = 'Loading...'; @@ -116,14 +102,14 @@ async function fetchResults(page = 1, updateHistory = true) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== 'page_size') { + if (value && key !== '_size' && key !== '_page') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('page', page.toString()); - params.append('page_size', pageSize); + params.append('_page', page.toString()); + params.append('_size', pageSize); try { const response = await fetch('{{ urls.path("-/allowed.json") }}?' + params.toString(), { @@ -136,7 +122,6 @@ async function fetchResults(page = 1, updateHistory = true) { const data = await response.json(); if (response.ok) { - currentData = data; displayResults(data); } else { displayError(data); @@ -164,8 +149,9 @@ function displayResults(data) { html += 'Resource Path'; html += 'Parent'; html += 'Child'; - html += 'Reason'; - html += 'Source Plugin'; + if (hasDebugPermission) { + html += 'Reason'; + } html += ''; html += ''; @@ -174,8 +160,14 @@ function displayResults(data) { html += `${escapeHtml(item.resource || '/')}`; html += `${escapeHtml(item.parent || '—')}`; html += `${escapeHtml(item.child || '—')}`; - html += `${escapeHtml(item.reason || '—')}`; - html += `${escapeHtml(item.source_plugin || '—')}`; + if (hasDebugPermission) { + // Display reason as JSON array + let reasonHtml = '—'; + if (item.reason && Array.isArray(item.reason)) { + reasonHtml = `${escapeHtml(JSON.stringify(item.reason))}`; + } + html += `${reasonHtml}`; + } html += ''; } @@ -188,13 +180,8 @@ function displayResults(data) { if (data.previous_url || data.next_url) { if (data.previous_url) { const prevLink = document.createElement('a'); - prevLink.href = '#'; + prevLink.href = data.previous_url; prevLink.textContent = '← Previous'; - prevLink.addEventListener('click', (e) => { - e.preventDefault(); - updateURL('allowed-form', data.page - 1); - fetchResults(data.page - 1, false); - }); pagination.appendChild(prevLink); } @@ -204,22 +191,14 @@ function displayResults(data) { if (data.next_url) { const nextLink = document.createElement('a'); - nextLink.href = '#'; + nextLink.href = data.next_url; nextLink.textContent = 'Next →'; - nextLink.addEventListener('click', (e) => { - e.preventDefault(); - updateURL('allowed-form', data.page + 1); - fetchResults(data.page + 1, false); - }); pagination.appendChild(nextLink); } } // Update raw JSON document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); - - // Scroll to results - resultsContainer.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); } function displayError(data) { @@ -230,20 +209,21 @@ function displayError(data) { resultsContent.innerHTML = `
Error: ${escapeHtml(data.error || 'Unknown error')}
`; document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); - - resultsContainer.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); } // Disable child input if parent is empty const parentInput = document.getElementById('parent'); const childInput = document.getElementById('child'); -childInput.addEventListener('focus', () => { +parentInput.addEventListener('input', () => { + childInput.disabled = !parentInput.value; if (!parentInput.value) { - alert('Please specify a parent resource first before filtering by child resource.'); - parentInput.focus(); + childInput.value = ''; } }); + +// Initialize disabled state +childInput.disabled = !parentInput.value; {% endblock %} diff --git a/datasette/templates/debug_autocomplete.html b/datasette/templates/debug_autocomplete.html new file mode 100644 index 00000000..380639a3 --- /dev/null +++ b/datasette/templates/debug_autocomplete.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} + +{% block title %}Debug autocomplete{% endblock %} + +{% block extra_head %} +{{ super() }} + +{% endblock %} + +{% block content %} +

Debug autocomplete

+ + +

+ + +

+

+ + +

+

+ + +{% if error %} +

{{ error }}

+{% elif autocomplete_url %} +

{{ database_name }} / {{ table_name }}

+ {% if label_column %} +

Label column: {{ label_column }}

+ {% else %} +

No label column detected. Results will use primary key values.

+ {% endif %} +
+ + + + +
+

Selected row

+ 
No row selected.
+ +{% else %} +

Suggested tables

+ {% if suggestions %} +

Showing up to five tables with a detected label column.

+ + + + + + + + + + {% for suggestion in suggestions %} + + + + + + {% endfor %} + +
DatabaseTableLabel column
{{ suggestion.database }}{{ suggestion.table }}{{ suggestion.label_column }}
+ {% else %} +

No tables with detected label columns found.

+ {% endif %} +

Scanned {{ scanned }} table{% if scanned != 1 %}s{% endif %}{% if reached_scan_limit %}; stopped at the 100 table scan limit{% endif %}.

+{% endif %} + +{% endblock %} diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index 2e077327..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,67 +1,47 @@ {% extends "base.html" %} -{% block title %}Permission Check{% endblock %} +{% block title %}Explain a permission decision{% endblock %} {% block extra_head %} - + +{% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} {% block content %} +

Explain a permission decision

-

Permission Check

+{% set current_tab = "check" %} +{% include "_permissions_debug_tabs.html" %} -

Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

+

Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

-{% if request.actor %} -

Current actor: {{ request.actor.get("id", "anonymous") }}

-{% else %} -

Current actor: anonymous (not logged in)

-{% endif %} +
+
+
+ + + Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. +
- -
- - - The permission action to check -
+
+ + + The operation to evaluate +
-
- - - For database-level permissions, specify the database name -
+
+ + + The database or other parent resource +
-
- - - For table-level permissions, specify the table name (requires parent) -
+
+ + + The table, query or other child resource +
- -
+
+ +
+ +
+actionSelect.addEventListener('change', updateResourceFields); +(function initializeFromUrl() { + const params = populateFormFromURL(); + updateResourceFields(); + if (params.get('action')) { + performCheck(); + } +})(); + {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html new file mode 100644 index 00000000..8b0cbbcf --- /dev/null +++ b/datasette/templates/debug_permissions_playground.html @@ -0,0 +1,150 @@ +{% extends "base.html" %} + +{% block title %}Permission activity{% endblock %} + +{% block extra_head %} +{% include "_permission_ui_styles.html" %} + +{% endblock %} + +{% block content %} +

Permission activity

+ +{% set current_tab = "permissions" %} +{% include "_permissions_debug_tabs.html" %} + +

Raw simulator

+ +

This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

+ +
+
+
+
+
+ + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ +
+ 

+    

+

+
+
+
+
Recent permission checks

+
+

+    {% if filter != "all" %}All{% else %}All{% endif %},
+    {% if filter != "exclude-yours" %}Exclude yours{% else %}Exclude yours{% endif %},
+    {% if filter != "only-yours" %}Only yours{% else %}Only yours{% endif %}
+

+
+{% if permission_checks %}
+
+    
+        
+            
+            
+            
+            
+            
+            
+        
+    
+    
+        {% for check in permission_checks %}
+            
+                
+                
+                
+                
+                
+                
+            
+        {% endfor %}
+        
+
WhenActionParentChildActorResult
{{ check.when.split('T', 1)[0] }}
{{ check.when.split('T', 1)[1].split('+', 1)[0].split('-', 1)[0].split('Z', 1)[0] }}		{{ check.action }}{{ check.parent or '—' }}{{ check.child or '—' }}{% if check.actor %}{{ check.actor|tojson }}{% else %}anonymous{% endif %}{% if check.result %}Allowed{% elif check.result is none %}No opinion{% else %}Denied{% endif %}

+{% else %}
+
No permission checks have been recorded yet.

+{% endif %}
+
+{% endblock %}
diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html
index d2d4c533..233c0e94 100644
--- a/datasette/templates/debug_rules.html
+++ b/datasette/templates/debug_rules.html
@@ -3,14 +3,16 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
-
+
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
 
 {% block content %}
+
Permission rules

 
-
Permission Rules

+{% set current_tab = "rules" %}
+{% include "_permissions_debug_tabs.html" %}
 
 
Use this tool to view the permission rules that allow the current actor to access resources for a given permission action. It queries the /-/rules.json API endpoint.

 
@@ -21,13 +23,13 @@
 {% endif %}
 
 

-    

+    
         

             
             
             The permission action to check
@@ -35,7 +37,7 @@
 
         

             
-            
+            
             Number of results per page (max 200)
         

 
@@ -68,35 +70,18 @@ const resultsContent = document.getElementById('results-content');
 const resultsCount = document.getElementById('results-count');
 const pagination = document.getElementById('pagination');
 const submitBtn = document.getElementById('submit-btn');
-let currentData = null;
-
-form.addEventListener('submit', async (ev) => {
-    ev.preventDefault();
-    updateURL('rules-form', 1);
-    await fetchResults(1, false);
-});
-
-// Handle browser back/forward
-window.addEventListener('popstate', () => {
-    const params = populateFormFromURL();
-    const action = params.get('action');
-    const page = params.get('page');
-    if (action) {
-        fetchResults(page ? parseInt(page) : 1, false);
-    }
-});
 
 // Populate form on initial load
 (function() {
     const params = populateFormFromURL();
     const action = params.get('action');
-    const page = params.get('page');
+    const page = params.get('_page');
     if (action) {
-        fetchResults(page ? parseInt(page) : 1, false);
+        fetchResults(page ? parseInt(page) : 1);
     }
 })();
 
-async function fetchResults(page = 1, updateHistory = true) {
+async function fetchResults(page = 1) {
     submitBtn.disabled = true;
     submitBtn.textContent = 'Loading...';
 
@@ -104,14 +89,14 @@ async function fetchResults(page = 1, updateHistory = true) {
     const params = new URLSearchParams();
 
     for (const [key, value] of formData.entries()) {
-        if (value && key !== 'page_size') {
+        if (value && key !== '_size' && key !== '_page') {
             params.append(key, value);
         }
     }
 
     const pageSize = document.getElementById('page_size').value || '50';
-    params.append('page', page.toString());
-    params.append('page_size', pageSize);
+    params.append('_page', page.toString());
+    params.append('_size', pageSize);
 
     try {
         const response = await fetch('{{ urls.path("-/rules.json") }}?' + params.toString(), {
@@ -124,7 +109,6 @@ async function fetchResults(page = 1, updateHistory = true) {
         const data = await response.json();
 
         if (response.ok) {
-            currentData = data;
             displayResults(data);
         } else {
             displayError(data);
@@ -153,8 +137,8 @@ function displayResults(data) {
         html += 'Resource Path';
         html += 'Parent';
         html += 'Child';
-        html += 'Reason';
         html += 'Source Plugin';
+        html += 'Reason';
         html += '';
         html += '';
 
@@ -169,8 +153,8 @@ function displayResults(data) {
             html += `${escapeHtml(item.resource || '/')}`;
             html += `${escapeHtml(item.parent || '—')}`;
             html += `${escapeHtml(item.child || '—')}`;
-            html += `${escapeHtml(item.reason || '—')}`;
             html += `${escapeHtml(item.source_plugin || '—')}`;
+            html += `${escapeHtml(item.reason || '—')}`;
             html += '';
         }
 
@@ -183,13 +167,8 @@ function displayResults(data) {
     if (data.previous_url || data.next_url) {
         if (data.previous_url) {
             const prevLink = document.createElement('a');
-            prevLink.href = '#';
+            prevLink.href = data.previous_url;
             prevLink.textContent = '← Previous';
-            prevLink.addEventListener('click', (e) => {
-                e.preventDefault();
-                updateURL('rules-form', data.page - 1);
-                fetchResults(data.page - 1, false);
-            });
             pagination.appendChild(prevLink);
         }
 
@@ -199,22 +178,14 @@ function displayResults(data) {
 
         if (data.next_url) {
             const nextLink = document.createElement('a');
-            nextLink.href = '#';
+            nextLink.href = data.next_url;
             nextLink.textContent = 'Next →';
-            nextLink.addEventListener('click', (e) => {
-                e.preventDefault();
-                updateURL('rules-form', data.page + 1);
-                fetchResults(data.page + 1, false);
-            });
             pagination.appendChild(nextLink);
         }
     }
 
     // Update raw JSON
     document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
-
-    // Scroll to results
-    resultsContainer.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
 }
 
 function displayError(data) {
@@ -225,8 +196,6 @@ function displayError(data) {
     resultsContent.innerHTML = `
Error: ${escapeHtml(data.error || 'Unknown error')}
`;
 
     document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
-
-    resultsContainer.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
 }
 
 
diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html
new file mode 100644
index 00000000..592577f8
--- /dev/null
+++ b/datasette/templates/execute_write.html
@@ -0,0 +1,337 @@
+{% extends "base.html" %}
+
+{% block title %}Write to this database{% endblock %}
+
+{% block extra_head %}
+{{- super() -}}
+{% include "_codemirror.html" %}
+
+{% include "_execute_write_analysis_styles.html" %}
+{% include "_sql_parameter_styles.html" %}
+{% endblock %}
+
+{% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %}
+
+{% block crumbs %}
+{{ crumbs.nav(request=request, database=database) }}
+{% endblock %}
+
+{% block content %}
+
+
Write to this database

+
+
Execute SQL to insert, update or delete rows in this database.

+
+{% if execution_message %}
+    
{{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

+{% endif %}
+
+{% if execute_write_returns_rows %}
+    
Returned rows

+    {% if execute_write_truncated %}
+        
Only the first {{ "{:,}".format(execute_write_display_rows|length) }} returned rows are shown.

+    {% endif %}
+    {% set columns = execute_write_columns %}
+    {% set display_rows = execute_write_display_rows %}
+    {% set show_zero_results = true %}
+    {% include "_query_results.html" %}
+{% endif %}
+
+
+    {% if write_create_table_template_sql or write_template_tables %}
+        

+            

+                Start with a template
+                

+                    {% if write_create_table_template_sql %}
+                        
+                    {% endif %}
+                    {% if write_template_tables %}
+                        
+                        
+                        {% for operation in write_template_operations %}
+                            
+                        {% endfor %}
+                    {% endif %}
+                

+            

+        

+    {% else %}
+        
There are no tables that you can currently edit.

+    {% endif %}
+
+    

+
+    {% set sql_parameters_section_id = "execute-write-parameters-section" %}
+    {% set sql_parameters_allow_expand = true %}
+    {% include "_sql_parameters.html" %}
+
+    

+        
Query operations

+        {% if analysis_error %}
+            
{{ analysis_error }}

+        {% elif analysis_rows %}
+            

+                
+                    
+                        
+                        
+                        
+                        
+                        
+                    
+                
+                
+                    {% for row in analysis_rows %}
+                        
+                            
+                            
+                            
+                            
+                            
+                        
+                    {% endfor %}
+                
+            
OperationDatabaseTableRequired permissionAllowed
{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}

+        {% else %}
+            
Analysis will show each affected table and required permission.

+        {% endif %}
+    

+
+    

+        
+        {{ execute_disabled_reason or "" }}
+        {% if save_query_url %}Save this query{% endif %}
+    

+
+
+{% include "_codemirror_foot.html" %}
+{% include "_sql_parameter_scripts.html" %}
+{% include "_execute_write_analysis_scripts.html" %}
+
+
+
+{% if write_create_table_template_sql or write_template_tables %}
+
+{% endif %}
+
+{% endblock %}
diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html
index c8fc642a..a99870e6 100644
--- a/datasette/templates/logout.html
+++ b/datasette/templates/logout.html
@@ -10,7 +10,6 @@
 
 

     

-        
         
     

 

diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html
index 2940cd69..891cf915 100644
--- a/datasette/templates/messages_debug.html
+++ b/datasette/templates/messages_debug.html
@@ -19,7 +19,6 @@
                 
             
         

-        
         
     

 
diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html
index 7770f7d4..075c0117 100644
--- a/datasette/templates/patterns.html
+++ b/datasette/templates/patterns.html
@@ -2,7 +2,7 @@
 
 
     Datasette: Pattern Portfolio
-    
+    
     
     
     
@@ -11,7 +11,7 @@