diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..84e574fd --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,4 @@ +# Applying Black +35d6ee2790e41e96f243c1ff58be0c9c0519a8ce +368638555160fb9ac78f462d0f79b1394163fa30 +2b344f6a34d2adaa305996a1a580ece06397f6e4 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b969c4c1..88bb03b1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,7 @@ updates: schedule: interval: daily time: "13:00" - open-pull-requests-limit: 10 - ignore: - - dependency-name: black - versions: - - 21.4b0 - - 21.4b1 + groups: + python-packages: + patterns: + - "*" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 1ae96e89..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -1,67 +1,126 @@ name: Deploy latest.datasette.io on: + workflow_dispatch: push: branches: - - main + - main + # - 1.0-dev + +permissions: + contents: read jobs: deploy: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.13" + cache: pip - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] - python -m pip install -e .[docs] + python -m pip install . --group dev python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests if: ${{ github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - - name: Build fixtures.db - run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db + - name: Build fixtures.db and other files needed to deploy the demo + run: |- + python tests/fixtures.py \ + fixtures.db \ + fixtures-config.json \ + fixtures-metadata.json \ + plugins \ + --extra-db-filename extra_database.db - name: Build docs.db if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs - sphinx-build -b xml . _build + DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@master + - name: Set up the alternate-route demo + run: | + echo ' + from datasette import hookimpl + + @hookimpl + def startup(datasette): + db = datasette.get_database("fixtures2") + db.route = "alternative-route" + ' > plugins/alternative_route.py + cp fixtures.db fixtures2.db + - name: And the counters writable stored query demo + run: | + cat > plugins/counters.py < metadata.json + # cat metadata.json + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 with: - version: '275.0.0' - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Deploy to Cloud Run + env: + LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} run: |- gcloud config set run/region us-central1 gcloud config set project datasette-222320 export SUFFIX="-${GITHUB_REF#refs/heads/}" export SUFFIX=${SUFFIX#-main} - datasette publish cloudrun fixtures.db extra_database.db \ - -m fixtures.json \ + # Replace 1.0 with one-dot-zero in SUFFIX + export SUFFIX=${SUFFIX//1.0/one-dot-zero} + datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ + -m fixtures-metadata.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ - --install=pysqlite3-binary \ - --service "datasette-latest$SUFFIX" + --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \ + --install 'datasette-ephemeral-tables>=0.2.2' \ + --service "datasette-latest$SUFFIX" \ + --secret $LATEST_DATASETTE_SECRET - name: Deploy to docs as well (only for main) if: ${{ github.ref == 'refs/heads/main' }} run: |- diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000..b8fb8aaa --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,16 @@ +name: Read the Docs Pull Request Preview +on: + pull_request: + types: + - opened + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "datasette" diff --git a/.github/workflows/mirror-master-and-main.yml b/.github/workflows/mirror-master-and-main.yml deleted file mode 100644 index 8418df40..00000000 --- a/.github/workflows/mirror-master-and-main.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Mirror "master" and "main" branches -on: - push: - branches: - - master - - main - -jobs: - mirror: - runs-on: ubuntu-latest - steps: - - name: Mirror to "master" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: master - force: false - - name: Mirror to "main" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: main - force: false diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000..5275ddef --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,48 @@ +name: Playwright + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + browser: [chromium, firefox, webkit] + steps: + - uses: actions/checkout@v6 + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: "3.14" + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml + - name: Cache uv + uses: actions/cache@v5 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py3.14-uv- + - name: Cache Playwright browsers + uses: actions/cache@v5 + with: + path: ~/.cache/ms-playwright/ + key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-playwright-${{ matrix.browser }}- + - name: Install uv + run: python -m pip install uv + - name: Install dependencies + run: uv sync --group dev --group playwright + - name: Install ${{ matrix.browser }} + run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }} + - name: Run Playwright tests + run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 9dfe7ee0..735e14e9 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -2,13 +2,16 @@ name: Check JavaScript for conformance with Prettier on: [push] +permissions: + contents: read + jobs: prettier: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v2 - - uses: actions/cache@v2 + uses: actions/checkout@v6 + - uses: actions/cache@v5 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3cfc67da..87300593 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,28 +4,26 @@ on: release: types: [created] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev - name: Run tests run: | pytest @@ -33,63 +31,55 @@ jobs: deploy: runs-on: ubuntu-latest needs: [test] + environment: release + permissions: + id-token: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: - python-version: '3.10' - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + python-version: '3.13' + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install setuptools wheel twine - - name: Publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + pip install setuptools wheel build + - name: Build run: | - python setup.py sdist bdist_wheel - twine upload dist/* + python -m build + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: python-version: '3.10' - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - python -m pip install -e .[docs] + python -m pip install . --group dev python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- cd docs - sphinx-build -b xml . _build + DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@master + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 with: - version: '275.0.0' - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 - name: Deploy stable-docs.datasette.io to Cloud Run run: |- gcloud config set run/region us-central1 @@ -102,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index 9a3969f0..e622ef4c 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -6,11 +6,14 @@ on: version_tag: description: Tag to build and push +permissions: + contents: read + jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 2e24d3eb..9a808194 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -2,26 +2,26 @@ name: Check spelling in documentation on: [push, pull_request] +permissions: + contents: read + jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' - name: Install dependencies run: | - pip install -e '.[docs]' + pip install . --group dev - name: Check spelling run: | + codespell README.md --ignore-words docs/codespell-ignore-words.txt codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + codespell tests --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml new file mode 100644 index 00000000..59b5fbc0 --- /dev/null +++ b/.github/workflows/stable-docs.yml @@ -0,0 +1,76 @@ +name: Update Stable Docs + +on: + release: + types: [published] + push: + branches: + - main + +permissions: + contents: write + +jobs: + update_stable_docs: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 # We need all commits to find docs/ changes + - name: Set up Git user + run: | + git config user.name "Automated" + git config user.email "actions@users.noreply.github.com" + - name: Create stable branch if it does not yet exist + run: | + if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then + # Make sure we have all tags locally + git fetch --tags --quiet + + # Latest tag that is just numbers and dots (optionally prefixed with 'v') + # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. + LATEST_RELEASE=$( + git tag -l --sort=-v:refname \ + | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ + | head -n1 + ) + + git checkout -b stable + + # If there are any stable releases, copy docs/ from the most recent + if [ -n "$LATEST_RELEASE" ]; then + rm -rf docs/ + git checkout "$LATEST_RELEASE" -- docs/ || true + fi + + git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" + git push -u origin stable + fi + - name: Handle Release + if: github.event_name == 'release' && !github.event.release.prerelease + run: | + git fetch --all + git checkout stable + git reset --hard ${GITHUB_REF#refs/tags/} + git push origin stable --force + - name: Handle Commit to Main + if: contains(github.event.head_commit.message, '!stable-docs') + run: | + git fetch origin + git checkout -b stable origin/stable + # Get the list of modified files in docs/ from the current commit + FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) + # Check if the list of files is non-empty + if [[ -n "$FILES" ]]; then + # Checkout those files to the stable branch to over-write with their contents + for FILE in $FILES; do + git checkout ${{ github.sha }} -- $FILE + done + git add docs/ + git commit -m "Doc changes from ${{ github.sha }}" + git push origin stable + else + echo "No changes to docs/ in this commit." + exit 0 + fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1d1cf332..c514048e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -7,33 +7,31 @@ on: pull_request: branches: - main +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] + python -m pip install . --group dev python -m pip install pytest-cov - name: Run tests run: |- ls -lah cat .coveragerc - pytest --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term + pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x ls -lah - name: Upload coverage report uses: codecov/codecov-action@v1 diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml new file mode 100644 index 00000000..5162c47a --- /dev/null +++ b/.github/workflows/test-pyodide.yml @@ -0,0 +1,33 @@ +name: Test in Pyodide with shot-scraper + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Set up Python 3.10 + uses: actions/setup-python@v6 + with: + python-version: "3.10" + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' + - name: Cache Playwright browsers + uses: actions/cache@v5 + with: + path: ~/.cache/ms-playwright/ + key: ${{ runner.os }}-browsers + - name: Install Playwright dependencies + run: | + pip install shot-scraper build + shot-scraper install + - name: Run test + run: | + ./test-in-pyodide-with-shot-scraper.sh diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml new file mode 100644 index 00000000..23fce459 --- /dev/null +++ b/.github/workflows/test-sqlite-support.yml @@ -0,0 +1,53 @@ +name: Test SQLite versions + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.platform }} + continue-on-error: true + strategy: + matrix: + platform: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + sqlite-version: [ + #"3", # latest version + "3.46", + #"3.45", + #"3.27", + #"3.26", + "3.25", + #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now + #"3.24", # 2018-06-04, added UPSERT support + #"3.23.1" # 2018-04-10, before UPSERT + ] + steps: + - uses: actions/checkout@v6 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml + - name: Set up SQLite ${{ matrix.sqlite-version }} + uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + with: + version: ${{ matrix.sqlite-version }} + cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" + - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" + - run: echo $LD_LIBRARY_PATH + - name: Build extension for --load-extension test + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so) + - name: Install dependencies + run: | + pip install . --group dev + pip freeze + - name: Run tests + run: | + pytest -n auto -m "not serial" + pytest -m "serial" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 478e1f34..9e47db6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,34 +1,53 @@ name: Test -on: [push] +on: [push, pull_request] + +permissions: + contents: read jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml + - name: Build extension for --load-extension test + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev pip freeze - name: Run tests run: | pytest -n auto -m "not serial" pytest -m "serial" + # And the test that exceeds a localhost HTTPS server + tests/test_datasette_https_server.sh + - name: Black + run: | + black --version + black --check . + - name: Ruff + run: ruff check datasette tests - name: Check if cog needs to be run run: | cog --check docs/*.rst + - name: Check if blacken-docs needs to be run + run: | + # This fails on syntax errors, or a diff was applied + blacken-docs -l 60 docs/*.rst + - name: Test DATASETTE_LOAD_PLUGINS + run: | + pip install datasette-init datasette-json-html + tests/test-datasette-load-plugins.sh diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index 46be117e..a033cd92 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -3,10 +3,13 @@ name: tmate session mac on: workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 02e7bd33..72af1eec 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -3,10 +3,16 @@ name: tmate session on: workflow_dispatch: +permissions: + contents: read + models: read + jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 066009f0..8c058692 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,18 @@ build-metadata.json datasets.json +.playwright-mcp + scratchpad .vscode +uv.lock +data.db + +# test databases +*.db + # We don't use Pipfile, so ignore them Pipfile Pipfile.lock @@ -118,3 +126,11 @@ ENV/ .DS_Store node_modules .*.swp + +# In case someone compiled tests/ext.c for test_load_extensions, don't +# include it in source control. +tests/*.dylib +tests/*.so +tests/*.dll + +.idea diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e157fb9c..8b3e54aa 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,16 +1,17 @@ version: 2 -build: - os: ubuntu-20.04 - tools: - python: "3.9" - sphinx: - configuration: docs/conf.py + configuration: docs/conf.py -python: - install: - - method: pip - path: . - extra_requirements: - - docs +build: + os: ubuntu-24.04 + tools: + python: "3.13" + jobs: + install: + - pip install --upgrade pip + - pip install . --group dev + +formats: +- pdf +- epub diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..14d4c567 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +`swillison+datasette-code-of-conduct@gmail.com`. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/Dockerfile b/Dockerfile index 42f5529b..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-slim-bullseye as build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..5fcd9afd --- /dev/null +++ b/Justfile @@ -0,0 +1,76 @@ +export DATASETTE_SECRET := "not_a_secret" + +# Run tests and linters +@default: test lint + +# Setup project +@init: + uv sync + +# Run pytest with supplied options +@test *options: init + uv run pytest -n auto {{options}} + +# Install Playwright browser support, Chromium by default +@playwright-install browser="chromium": + uv run --group playwright playwright install {{browser}} + +# Install all Playwright browsers used by the test suite +@playwright-install-all: + uv run --group playwright playwright install chromium firefox webkit + +# Run Playwright tests, Chromium by default +@playwright browser="chromium" *options: + uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}} + +# Run Playwright tests against all supported browsers +@playwright-all *options: + uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}} + +@codespell: + uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + uv run codespell tests --ignore-words docs/codespell-ignore-words.txt + +# Run linters: black, ruff, cog +@lint: codespell + uv run black datasette tests --check + uv run ruff check datasette tests + uv run cog --check README.md docs/*.rst + +# Apply ruff fixes +@fix: + uv run ruff check --fix datasette tests + +# Rebuild docs with cog +@cog: + uv run cog -r README.md docs/*.rst + +# Serve live docs on localhost:8000 +@docs: cog blacken-docs + uv run make -C docs livehtml + +# Build docs as static HTML +@docs-build: cog blacken-docs + rm -rf docs/_build && cd docs && uv run make html + +# Apply Black +@black: + uv run black datasette tests + +# Apply blacken-docs +@blacken-docs: + uv run blacken-docs -l 60 docs/*.rst + +# Apply prettier +@prettier: + npm run fix + +# Format code with both black and prettier +@format: black prettier blacken-docs + +@serve *options: + uv run sqlite-utils create-database data.db + uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + uv run python -m datasette data.db --root --reload {{options}} diff --git a/README.md b/README.md index 557d9290..393e8e5c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) +[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord) *An open source multi-tool for exploring and publishing data* @@ -14,14 +15,14 @@ Datasette is a tool for exploring and publishing data. It helps people take data Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world. -[Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch). +[Explore a demo](https://datasette.io/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out [on GitHub Codespaces](https://github.com/datasette/datasette-studio). * [datasette.io](https://datasette.io/) is the official project website * Latest [Datasette News](https://datasette.io/news) * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples -* Live demo of current main: https://latest.datasette.io/ -* Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) +* Live demo of current `main` branch: https://latest.datasette.io/ +* Questions, feedback or want to talk about the project? Join our [Discord](https://datasette.io/discord) Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. @@ -35,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.7 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage @@ -47,7 +48,7 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce Use Chrome on OS X? You can run datasette against your browser history like so: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History + datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: @@ -84,3 +85,7 @@ Or: This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API. See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details. + +## Datasette Lite + +[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md). diff --git a/datasette/__init__.py b/datasette/__init__.py index faa36051..eb18e59e 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,5 +1,9 @@ +from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa -from datasette.utils.asgi import Forbidden, NotFound, Response # noqa +from datasette.events import Event # noqa +from datasette.tokens import TokenHandler, TokenRestrictions # noqa +from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa +from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa from .hookspecs import hookspec # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py new file mode 100644 index 00000000..103c616d --- /dev/null +++ b/datasette/_pytest_plugin.py @@ -0,0 +1,123 @@ +""" +Pytest plugin that automatically closes any Datasette instances constructed +during a pytest test — both in the test body and in function-scoped +fixtures. Instances constructed by session-, module-, class- or package- +scoped fixtures are left alone, because other tests in the session will +still want to use them. + +Registered as a pytest11 entry point in pyproject.toml so that downstream +projects using Datasette get the same FD-safety net for their own tests. + +Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the +equivalent ini file). +""" + +from __future__ import annotations + +import contextvars +import weakref + +import pytest + +_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "datasette_active_instances", default=None +) + +_original_init = None + + +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init + + +def pytest_configure(config): + if _enabled(config): + _install_tracking() + + +def pytest_addoption(parser): + parser.addini( + "datasette_autoclose", + help=( + "Automatically close Datasette instances created inside test " + "bodies and function-scoped fixtures (default: true)." + ), + default="true", + ) + + +def _enabled(config) -> bool: + value = config.getini("datasette_autoclose") + if isinstance(value, bool): + return value + return str(value).strip().lower() not in ("false", "0", "no", "off") + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + """Track Datasette instances across setup, call and teardown; close at end.""" + if not _enabled(item.config): + yield + return + refs: list[weakref.ref] = [] + token = _active_instances.set(refs) + try: + yield + finally: + _active_instances.reset(token) + for ref in reversed(refs): + ds = ref() + if ds is None: + continue + try: + ds.close() + except Exception as e: + item.warn( + pytest.PytestUnraisableExceptionWarning( + f"Error closing Datasette instance: {e!r}" + ) + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request): + """Exempt instances created by non-function-scoped fixtures. + + Session-, module-, class- and package-scoped fixtures produce Datasette + instances that must survive beyond the current test — other tests in + the session will still use them. When such a fixture creates one or + more Datasette instances during its setup, we snapshot the tracking + list before the fixture runs and subtract off any instances that were + added during its setup, so they don't get closed at test teardown. + """ + refs = _active_instances.get() + if refs is None: + yield + return + before_ids = {id(ref) for ref in refs} + yield + if fixturedef.scope != "function": + new_refs = [ref for ref in refs if id(ref) not in before_ids] + for new_ref in new_refs: + try: + refs.remove(new_ref) + except ValueError: + pass diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 15ecd331..368213af 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,6 +1,6 @@ from datasette import hookimpl from itsdangerous import BadSignature -import baseconv +from datasette.utils import baseconv import time diff --git a/datasette/app.py b/datasette/app.py index 2907d90e..9c9b7de4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,81 +1,151 @@ +from __future__ import annotations + import asyncio -import asgi_csrf +import contextvars +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence + +if TYPE_CHECKING: + from datasette.permissions import Resource + from datasette.tokens import TokenRestrictions import collections +import dataclasses import datetime import functools import glob -import hashlib import httpx +import importlib.metadata import inspect from itsdangerous import BadSignature import json import os -import pkg_resources import re import secrets import sys import threading -import traceback +import time +import types import urllib.parse from concurrent import futures from pathlib import Path from markupsafe import Markup, escape from itsdangerous import URLSafeSerializer -from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader +from jinja2 import ( + ChoiceLoader, + Environment, + FileSystemLoader, + pass_context, + PrefixLoader, +) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -import uvicorn -from .views.base import DatasetteError, ureg -from .views.database import DatabaseDownload, DatabaseView +from .events import Event +from .column_types import SQLiteType +from . import stored_queries, write_sql +from .views import Context +from .views.database import ( + database_download, + DatabaseView, + QueryView, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + QueryEditView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) from .views.index import IndexView from .views.special import ( JsonDataView, PatternPortfolioView, + AutocompleteDebugView, AuthTokenView, + ApiExplorerView, + CreateTokenView, LogoutView, AllowDebugView, PermissionsDebugView, MessagesDebugView, + AllowedResourcesView, + PermissionRulesView, + PermissionCheckView, + JumpView, + InstanceSchemaView, + DatabaseSchemaView, + TableSchemaView, ) -from .views.table import RowView, TableView +from .views.table import ( + TableAutocompleteView, + TableInsertView, + TableUpsertView, + TableSetColumnTypeView, + TableDropView, + TableFragmentView, + table_view, +) +from .views.row import RowView, RowDeleteView, RowUpdateView from .renderer import json_renderer from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( + PaginatedResources, PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, - add_cors_headers, async_call_with_supported_arguments, await_me_maybe, + baseconv, call_with_supported_arguments, + detect_json1, display_actor, escape_css_string, escape_sqlite, find_spatialite, format_bytes, module_from_path, + move_plugins_and_allow, + move_table_config, parse_metadata, resolve_env_secrets, + resolve_routes, + sha256_file, + tilde_decode, + tilde_encode, to_css_class, + urlsafe_components, + redact_keys, + row_sql_params_pks, ) from .utils.asgi import ( AsgiLifespan, - Base400, Forbidden, NotFound, + DatabaseNotFound, + TableNotFound, + RowNotFound, Request, Response, + AsgiRunOnFirstRequest, asgi_static, asgi_send, asgi_send_file, - asgi_send_html, - asgi_send_json, asgi_send_redirect, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -85,16 +155,44 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -try: - import rich -except ImportError: - rich = None +from .resources import DatabaseResource, TableResource app_root = Path(__file__).parent.parent + +# Context variable to track when code is executing within a datasette.client request +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + + +class _DatasetteClientContext: + """Context manager to mark code as executing within a datasette.client request.""" + + def __enter__(self): + self.token = _in_datasette_client.set(True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + _in_datasette_client.reset(self.token) + return False + + +@dataclasses.dataclass +class PermissionCheck: + """Represents a logged permission check for debugging purposes.""" + + when: str + actor: Dict[str, Any] | None + action: str + parent: str | None + child: str | None + result: bool + + # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 SQLITE_LIMIT_ATTACHED = 10 +INTERNAL_DB_NAME = "__INTERNAL__" + Setting = collections.namedtuple("Setting", ("name", "default", "help")) SETTINGS = ( Setting("default_page_size", 100, "Default page size for the table view"), @@ -103,6 +201,11 @@ SETTINGS = ( 1000, "Maximum rows that can be returned from a table or custom query", ), + Setting( + "max_insert_rows", + 100, + "Maximum rows that can be inserted at a time using the bulk insert API", + ), Setting( "num_sql_threads", 3, @@ -118,11 +221,6 @@ SETTINGS = ( 50, "Time limit for calculating a suggested facet", ), - Setting( - "hash_urls", - False, - "Include DB file contents hash in URLs, for far-future caching", - ), Setting( "allow_facet", True, @@ -133,17 +231,27 @@ SETTINGS = ( True, "Allow users to download the original SQLite database files", ), + Setting( + "allow_signed_tokens", + True, + "Allow users to create and use signed API tokens", + ), + Setting( + "default_allow_sql", + True, + "Allow anyone to run arbitrary SQL queries", + ), + Setting( + "max_signed_tokens_ttl", + 0, + "Maximum allowed expiry time for signed API tokens", + ), Setting("suggest_facets", True, "Calculate and display suggested facets"), Setting( "default_cache_ttl", 5, "Default HTTP cache TTL (used in Cache-Control: max-age= header)", ), - Setting( - "default_cache_ttl_hashed", - 365 * 24 * 60 * 60, - "Default HTTP cache TTL for hashed URL pages", - ), Setting("cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)"), Setting( "allow_csv_stream", @@ -177,21 +285,103 @@ SETTINGS = ( ), Setting("base_url", "/", "Datasette URLs should use this base path"), ) - +_HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead" +OBSOLETE_SETTINGS = { + "hash_urls": _HASH_URLS_REMOVED, + "default_cache_ttl_hashed": _HASH_URLS_REMOVED, +} DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" +DEFAULT_NOT_SET = object() + + +ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) + + +def _permission_cache_key(actor, action, parent, child): + # Key on the full serialized actor so actors differing in any field + # (e.g. token restrictions) never share cache entries + actor_key = ( + json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None + ) + return (actor_key, action, parent, child) + async def favicon(request, send): await asgi_send_file( send, str(FAVICON_PATH), content_type="image/png", - headers={"Cache-Control": "max-age=3600, immutable, public"}, + headers={"Cache-Control": "max-age=3600, public"}, ) +ResolvedTable = collections.namedtuple("ResolvedTable", ("db", "table", "is_view")) +ResolvedRow = collections.namedtuple( + "ResolvedRow", ("db", "table", "sql", "params", "pks", "pk_values", "row") +) + + +def _to_string(value): + if isinstance(value, str): + return value + else: + return json.dumps(value, default=str) + + +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + +def _resolve_static_asset_path(root_path, path): + root = Path(root_path).resolve() + full_path = (root / path).resolve() + try: + full_path.relative_to(root) + except ValueError: + raise ValueError("Static asset path cannot escape static root") from None + return full_path + + +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", + "show_logout": "True if the logout link should be shown in the navigation menu", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` diff --git a/datasette/templates/_codemirror.html b/datasette/templates/_codemirror.html index a17eaf9b..75c16168 100644 --- a/datasette/templates/_codemirror.html +++ b/datasette/templates/_codemirror.html @@ -1,14 +1,16 @@ - - - - - + + diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index ee09cff1..a624c8a4 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -1,38 +1,42 @@ diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html new file mode 100644 index 00000000..bd1ff0da --- /dev/null +++ b/datasette/templates/_crumbs.html @@ -0,0 +1,15 @@ +{% macro nav(request, database=None, table=None) -%} +{% if crumb_items is defined %} + {% set items=crumb_items(request=request, database=database, table=table) %} + {% if items %} +

+ {% for item in items %} + {{ item.label }} + {% if not loop.last %} + / + {% endif %} + {% endfor %} +

+ {% endif %} +{% endif %} +{%- endmacro %} diff --git a/datasette/templates/_debug_common_functions.html b/datasette/templates/_debug_common_functions.html new file mode 100644 index 00000000..d988a2f3 --- /dev/null +++ b/datasette/templates/_debug_common_functions.html @@ -0,0 +1,50 @@ + diff --git a/datasette/templates/_description_source_license.html b/datasette/templates/_description_source_license.html index a2bc18f2..f852268f 100644 --- a/datasette/templates/_description_source_license.html +++ b/datasette/templates/_description_source_license.html @@ -1,6 +1,6 @@ -{% if metadata.description_html or metadata.description %} +{% if metadata.get("description_html") or metadata.get("description") %}
- {% if metadata.description_html %} + {% if metadata.get("description_html") %} {{ metadata.description_html|safe }} {% else %} {{ metadata.description }} diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html new file mode 100644 index 00000000..a19bae13 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_scripts.html @@ -0,0 +1,111 @@ + diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html new file mode 100644 index 00000000..165cfe9f --- /dev/null +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -0,0 +1,41 @@ + diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html new file mode 100644 index 00000000..570bb37e --- /dev/null +++ b/datasette/templates/_facet_results.html @@ -0,0 +1,28 @@ +
+ {% for facet_info in sorted_facet_results %} +
+

+ {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} + {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }} + + {% if facet_info.hideable %} + ✖ + {% endif %} +

+
    + {% for facet_value in facet_info.results %} + {% if not facet_value.selected %} +
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • + {% else %} +
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • + {% endif %} + {% endfor %} + {% if facet_info.truncated %} +
  • {% if request.args._facet_size != "max" -%} + …{% else -%}…{% endif %} +
    • + {% endif %} +
+
+ {% endfor %} +
diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html new file mode 100644 index 00000000..53a824f1 --- /dev/null +++ b/datasette/templates/_permission_ui_styles.html @@ -0,0 +1,145 @@ + diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html new file mode 100644 index 00000000..d7203c1e --- /dev/null +++ b/datasette/templates/_permissions_debug_tabs.html @@ -0,0 +1,54 @@ +{% if has_debug_permission %} +{% set query_string = '?' + request.query_string if request.query_string else '' %} + + + + +{% endif %} diff --git a/datasette/templates/_query_form_styles.html b/datasette/templates/_query_form_styles.html new file mode 100644 index 00000000..cf2dd42c --- /dev/null +++ b/datasette/templates/_query_form_styles.html @@ -0,0 +1,138 @@ + diff --git a/datasette/templates/_query_results.html b/datasette/templates/_query_results.html new file mode 100644 index 00000000..5e1e2f72 --- /dev/null +++ b/datasette/templates/_query_results.html @@ -0,0 +1,20 @@ +{% if display_rows %} +
+ + + {% for column in columns %}{% endfor %} + + + + {% for row in display_rows %} + + {% for column, td in zip(columns, row) %} + + {% endfor %} + + {% endfor %} + +
{{ column }}
{{ td }}
+{% elif show_zero_results %} +

0 results

+{% endif %} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html new file mode 100644 index 00000000..9b83889e --- /dev/null +++ b/datasette/templates/_sql_parameter_scripts.html @@ -0,0 +1,307 @@ + diff --git a/datasette/templates/_sql_parameter_styles.html b/datasette/templates/_sql_parameter_styles.html new file mode 100644 index 00000000..bc6838f5 --- /dev/null +++ b/datasette/templates/_sql_parameter_styles.html @@ -0,0 +1,58 @@ + diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html new file mode 100644 index 00000000..b5c1bde8 --- /dev/null +++ b/datasette/templates/_sql_parameters.html @@ -0,0 +1,10 @@ +{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %} +
+ {% if parameter_names %} +

Parameters

+ {% for parameter in parameter_names %} + {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} +

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

+ {% endfor %} + {% endif %} +
diff --git a/datasette/templates/_suggested_facets.html b/datasette/templates/_suggested_facets.html new file mode 100644 index 00000000..b80208c3 --- /dev/null +++ b/datasette/templates/_suggested_facets.html @@ -0,0 +1,3 @@ +

+ Suggested facets: {% for facet in suggested_facets %}{{ facet.name }}{% if facet.get("type") %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %} +

diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index 5332f831..171b6442 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,10 +1,12 @@ -{% if display_rows %} + +
+{% if display_columns %}
{% for column in display_columns %} - {% for row in display_rows %} - + {% for cell in row %} {% endfor %} @@ -29,6 +31,7 @@
+ {% if not column.sortable %} {{ column.name }} {% else %} @@ -20,7 +22,7 @@
{{ cell.value }}
-{% else %} +{% endif %} +{% if not display_rows %}

0 records

{% endif %} diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 0f1b30f0..1ecc92df 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -33,9 +33,12 @@ p.message-warning {

Debug allow rules

+{% set current_tab = "allow_debug" %} +{% include "_permissions_debug_tabs.html" %} +

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

-
+

diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html new file mode 100644 index 00000000..4927cb8d --- /dev/null +++ b/datasette/templates/api_explorer.html @@ -0,0 +1,208 @@ +{% extends "base.html" %} + +{% block title %}API Explorer{% endblock %} + +{% block extra_head %} + +{% endblock %} + +{% block content %} + +

API Explorer{% if private %} 🔒{% endif %}

+ +

Use this tool to try out the + {% if datasette_version %} + Datasette API. + {% else %} + Datasette API. + {% endif %} +

+
+ GET + +
+ + + +
+ +
+
+ POST +
+
+ + +
+
+ + +
+

+
+
+ + + + + +{% if example_links %} +

API endpoints

+
    + {% for database in example_links %} +
  • Database: {{ database.name }}
    • +
      + {% for link in database.links %} +
    • {{ link.path }} - {{ link.label }}
      • + {% endfor %} + {% for table in database.tables %} +
    • {{ table.name }} +
        + {% for link in table.links %} +
      • {{ link.path }} - {{ link.label }}
        • + {% endfor %} +
      +
      • + {% endfor %} +
    + {% endfor %} +
+{% endif %} + +{% endblock %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c3a71acb..18288439 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,14 +1,16 @@ - - +{% import "_crumbs.html" as crumbs with context %} + {% block title %}{% endblock %} - + {% for url in extra_css_urls %} - + {% endfor %} + + {% for url in extra_js_urls %} - + {% endfor %} {%- if alternate_url_json -%} @@ -17,9 +19,9 @@
-