diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 00000000..872aff71 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b0640ae8..0dfa5a60 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -1,11 +1,10 @@ name: Deploy latest.datasette.io on: - workflow_dispatch: push: branches: - main - # - 1.0-dev + - 1.0-dev permissions: contents: read @@ -15,30 +14,32 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v4 + # gcloud commmand breaks on higher Python versions, so stick with 3.9: with: - python-version: "3.13" - cache: pip + python-version: "3.9" + - uses: actions/cache@v3 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install . --group dev + python -m pip install -e .[test] + python -m pip install -e .[docs] python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests if: ${{ github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - - name: Build fixtures.db and other files needed to deploy the demo - run: |- - python tests/fixtures.py \ - fixtures.db \ - fixtures-config.json \ - fixtures-metadata.json \ - plugins \ - --extra-db-filename extra_database.db + - name: Build fixtures.db + run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db - name: Build docs.db if: ${{ github.ref == 'refs/heads/main' }} run: |- @@ -57,7 +58,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable stored query demo + - name: And the counters writable canned query demo run: | cat > plugins/counters.py < metadata.json - # cat metadata.json - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 + - name: Make some modifications to metadata.json + run: | + cat fixtures.json | \ + jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ + jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ + > metadata.json + cat metadata.json + - name: Set up Cloud Run + uses: google-github-actions/setup-gcloud@v0 with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + version: '318.0.0' + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Deploy to Cloud Run env: LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} @@ -113,11 +112,11 @@ jobs: # Replace 1.0 with one-dot-zero in SUFFIX export SUFFIX=${SUFFIX//1.0/one-dot-zero} datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ - -m fixtures-metadata.json \ + -m metadata.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \ + --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ --install 'datasette-ephemeral-tables>=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index b8fb8aaa..a54bd83a 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request: + pull_request_target: types: - opened diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 735e14e9..ded41040 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v6 - - uses: actions/cache@v5 + uses: actions/checkout@v2 + - uses: actions/cache@v2 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87300593..64a03a77 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,18 +12,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: pyproject.toml + - uses: actions/cache@v3 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[test]' - name: Run tests run: | pytest @@ -31,41 +36,50 @@ jobs: deploy: runs-on: ubuntu-latest needs: [test] - environment: release - permissions: - id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v4 with: - python-version: '3.13' - cache: pip - cache-dependency-path: pyproject.toml + python-version: '3.11' + - uses: actions/cache@v3 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-publish-pip- - name: Install dependencies run: | - pip install setuptools wheel build - - name: Build - run: | - python -m build + pip install setuptools wheel twine - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: - python-version: '3.10' - cache: pip - cache-dependency-path: pyproject.toml + python-version: '3.9' + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-publish-pip- - name: Install dependencies run: | - python -m pip install . --group dev + python -m pip install -e .[docs] python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- @@ -73,13 +87,12 @@ jobs: DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 + - name: Set up Cloud Run + uses: google-github-actions/setup-gcloud@v0 with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + version: '318.0.0' + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Deploy stable-docs.datasette.io to Cloud Run run: |- gcloud config set run/region us-central1 @@ -92,7 +105,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index e622ef4c..afe8d6b2 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 9a808194..722e5c68 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,19 +9,23 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v6 + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 with: - python-version: '3.11' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + python-version: 3.11 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[docs]' - name: Check spelling run: | codespell README.md --ignore-words docs/codespell-ignore-words.txt codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - codespell tests --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml deleted file mode 100644 index 59b5fbc0..00000000 --- a/.github/workflows/stable-docs.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Update Stable Docs - -on: - release: - types: [published] - push: - branches: - - main - -permissions: - contents: write - -jobs: - update_stable_docs: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 # We need all commits to find docs/ changes - - name: Set up Git user - run: | - git config user.name "Automated" - git config user.email "actions@users.noreply.github.com" - - name: Create stable branch if it does not yet exist - run: | - if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then - # Make sure we have all tags locally - git fetch --tags --quiet - - # Latest tag that is just numbers and dots (optionally prefixed with 'v') - # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. - LATEST_RELEASE=$( - git tag -l --sort=-v:refname \ - | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ - | head -n1 - ) - - git checkout -b stable - - # If there are any stable releases, copy docs/ from the most recent - if [ -n "$LATEST_RELEASE" ]; then - rm -rf docs/ - git checkout "$LATEST_RELEASE" -- docs/ || true - fi - - git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" - git push -u origin stable - fi - - name: Handle Release - if: github.event_name == 'release' && !github.event.release.prerelease - run: | - git fetch --all - git checkout stable - git reset --hard ${GITHUB_REF#refs/tags/} - git push origin stable --force - - name: Handle Commit to Main - if: contains(github.event.head_commit.message, '!stable-docs') - run: | - git fetch origin - git checkout -b stable origin/stable - # Get the list of modified files in docs/ from the current commit - FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) - # Check if the list of files is non-empty - if [[ -n "$FILES" ]]; then - # Checkout those files to the stable branch to over-write with their contents - for FILE in $FILES; do - git checkout ${{ github.sha }} -- $FILE - done - git add docs/ - git commit -m "Doc changes from ${{ github.sha }}" - git push origin stable - else - echo "No changes to docs/ in this commit." - exit 0 - fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index c514048e..bd720664 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,23 +15,28 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install . --group dev + python -m pip install -e .[test] python -m pip install pytest-cov - name: Run tests run: |- ls -lah cat .coveragerc - pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x + pytest --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term ls -lah - name: Upload coverage report uses: codecov/codecov-action@v1 diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5162c47a..bc9593a8 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,15 +12,15 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v3 - name: Set up Python 3.10 - uses: actions/setup-python@v6 + uses: actions/setup-python@v3 with: python-version: "3.10" cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + cache-dependency-path: '**/setup.py' - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v2 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml deleted file mode 100644 index 23fce459..00000000 --- a/.github/workflows/test-sqlite-support.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Test SQLite versions - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - test: - runs-on: ${{ matrix.platform }} - continue-on-error: true - strategy: - matrix: - platform: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - sqlite-version: [ - #"3", # latest version - "3.46", - #"3.45", - #"3.27", - #"3.26", - "3.25", - #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now - #"3.24", # 2018-06-04, added UPSERT support - #"3.23.1" # 2018-04-10, before UPSERT - ] - steps: - - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 - with: - version: ${{ matrix.sqlite-version }} - cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" - - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" - - run: echo $LD_LIBRARY_PATH - - name: Build extension for --load-extension test - run: |- - (cd tests && gcc ext.c -fPIC -shared -o ext.so) - - name: Install dependencies - run: | - pip install . --group dev - pip freeze - - name: Run tests - run: | - pytest -n auto -m "not serial" - pytest -m "serial" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1b2e9d2..656b0b1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,22 +10,27 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml + - uses: actions/cache@v3 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[test]' pip freeze - name: Run tests run: | @@ -33,16 +38,16 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Black + - name: Install docs dependencies on Python 3.9+ + if: matrix.python-version != '3.8' run: | - black --version - black --check . - - name: Ruff - run: ruff check datasette tests + pip install -e '.[docs]' - name: Check if cog needs to be run + if: matrix.python-version != '3.8' run: | cog --check docs/*.rst - name: Check if blacken-docs needs to be run + if: matrix.python-version != '3.8' run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index a033cd92..fcee0f21 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 72af1eec..9792245d 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -5,14 +5,11 @@ on: permissions: contents: read - models: read jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 12acd87e..277ff653 100644 --- a/.gitignore +++ b/.gitignore @@ -5,12 +5,6 @@ scratchpad .vscode -uv.lock -data.db - -# test databases -*.db - # We don't use Pipfile, so ignore them Pipfile Pipfile.lock @@ -129,6 +123,4 @@ node_modules # include it in source control. tests/*.dylib tests/*.so -tests/*.dll - -.idea \ No newline at end of file +tests/*.dll \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b3e54aa..5b30e75a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,17 +1,16 @@ version: 2 -sphinx: - configuration: docs/conf.py - build: - os: ubuntu-24.04 + os: ubuntu-20.04 tools: - python: "3.13" - jobs: - install: - - pip install --upgrade pip - - pip install . --group dev + python: "3.11" -formats: -- pdf -- epub +sphinx: + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/Justfile b/Justfile index 657881be..d349ec51 100644 --- a/Justfile +++ b/Justfile @@ -5,56 +5,37 @@ export DATASETTE_SECRET := "not_a_secret" # Setup project @init: - uv sync + pipenv run pip install -e '.[test,docs]' # Run pytest with supplied options -@test *options: init - uv run pytest -n auto {{options}} +@test *options: + pipenv run pytest {{options}} @codespell: - uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt - uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - uv run codespell tests --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, ruff, cog +# Run linters: black, flake8, mypy, cog @lint: codespell - uv run black datasette tests --check - uv run ruff check datasette tests - uv run cog --check README.md docs/*.rst - -# Apply ruff fixes -@fix: - uv run ruff check --fix datasette tests + pipenv run black . --check + pipenv run flake8 + pipenv run cog --check README.md docs/*.rst # Rebuild docs with cog @cog: - uv run cog -r README.md docs/*.rst + pipenv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: cog blacken-docs - uv run make -C docs livehtml - -# Build docs as static HTML -@docs-build: cog blacken-docs - rm -rf docs/_build && cd docs && uv run make html +@docs: cog + pipenv run blacken-docs -l 60 docs/*.rst + cd docs && pipenv run make livehtml # Apply Black @black: - uv run black datasette tests + pipenv run black . -# Apply blacken-docs -@blacken-docs: - uv run blacken-docs -l 60 docs/*.rst - -# Apply prettier -@prettier: - npm run fix - -# Format code with both black and prettier -@format: black prettier blacken-docs - -@serve *options: - uv run sqlite-utils create-database data.db - uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore - uv run python -m datasette data.db --root --reload {{options}} +@serve: + pipenv run sqlite-utils create-database data.db + pipenv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + pipenv run python -m datasette data.db --root --reload diff --git a/README.md b/README.md index 393e8e5c..93d81ae6 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) -[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord) +[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/ktd74dm5mw) *An open source multi-tool for exploring and publishing data* @@ -15,14 +15,14 @@ Datasette is a tool for exploring and publishing data. It helps people take data Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world. -[Explore a demo](https://datasette.io/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out [on GitHub Codespaces](https://github.com/datasette/datasette-studio). +[Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch). * [datasette.io](https://datasette.io/) is the official project website * Latest [Datasette News](https://datasette.io/news) * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples * Live demo of current `main` branch: https://latest.datasette.io/ -* Questions, feedback or want to talk about the project? Join our [Discord](https://datasette.io/discord) +* Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw) Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. diff --git a/datasette/__init__.py b/datasette/__init__.py index eb18e59e..271e09ad 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,7 +1,5 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa -from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenRestrictions # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py deleted file mode 100644 index 5fb6b473..00000000 --- a/datasette/_pytest_plugin.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Pytest plugin that automatically closes any Datasette instances constructed -during a pytest test — both in the test body and in function-scoped -fixtures. Instances constructed by session-, module-, class- or package- -scoped fixtures are left alone, because other tests in the session will -still want to use them. - -Registered as a pytest11 entry point in pyproject.toml so that downstream -projects using Datasette get the same FD-safety net for their own tests. - -Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the -equivalent ini file). -""" - -from __future__ import annotations - -import contextvars -import weakref - -import pytest - -from datasette.app import Datasette - -_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( - "datasette_active_instances", default=None -) - -_original_init = Datasette.__init__ - - -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) - - -Datasette.__init__ = _tracking_init - - -def pytest_addoption(parser): - parser.addini( - "datasette_autoclose", - help=( - "Automatically close Datasette instances created inside test " - "bodies and function-scoped fixtures (default: true)." - ), - default="true", - ) - - -def _enabled(config) -> bool: - value = config.getini("datasette_autoclose") - if isinstance(value, bool): - return value - return str(value).strip().lower() not in ("false", "0", "no", "off") - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - """Track Datasette instances across setup, call and teardown; close at end.""" - if not _enabled(item.config): - yield - return - refs: list[weakref.ref] = [] - token = _active_instances.set(refs) - try: - yield - finally: - _active_instances.reset(token) - for ref in reversed(refs): - ds = ref() - if ds is None: - continue - try: - ds.close() - except Exception as e: - item.warn( - pytest.PytestUnraisableExceptionWarning( - f"Error closing Datasette instance: {e!r}" - ) - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef, request): - """Exempt instances created by non-function-scoped fixtures. - - Session-, module-, class- and package-scoped fixtures produce Datasette - instances that must survive beyond the current test — other tests in - the session will still use them. When such a fixture creates one or - more Datasette instances during its setup, we snapshot the tracking - list before the fixture runs and subtract off any instances that were - added during its setup, so they don't get closed at test teardown. - """ - refs = _active_instances.get() - if refs is None: - yield - return - before_ids = {id(ref) for ref in refs} - yield - if fixturedef.scope != "function": - new_refs = [ref for ref in refs if id(ref) not in before_ids] - for new_ref in new_refs: - try: - refs.remove(new_ref) - except ValueError: - pass diff --git a/datasette/app.py b/datasette/app.py index 56b89789..fdec2c86 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,12 +1,6 @@ -from __future__ import annotations - import asyncio -import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List - -if TYPE_CHECKING: - from datasette.permissions import Resource - from datasette.tokens import TokenRestrictions +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union +import asgi_csrf import collections import dataclasses import datetime @@ -14,11 +8,11 @@ import functools import glob import hashlib import httpx -import importlib.metadata import inspect from itsdangerous import BadSignature import json import os +import pkg_resources import re import secrets import sys @@ -40,27 +34,9 @@ from jinja2 import ( from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -from .events import Event -from .column_types import SQLiteType -from . import stored_queries from .views import Context -from .views.database import ( - database_download, - DatabaseView, - TableCreateView, - QueryView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) +from .views.base import ureg +from .views.database import database_download, DatabaseView, TableCreateView from .views.index import IndexView from .views.special import ( JsonDataView, @@ -72,18 +48,10 @@ from .views.special import ( AllowDebugView, PermissionsDebugView, MessagesDebugView, - AllowedResourcesView, - PermissionRulesView, - PermissionCheckView, - JumpView, - InstanceSchemaView, - DatabaseSchemaView, - TableSchemaView, ) from .views.table import ( TableInsertView, TableUpsertView, - TableSetColumnTypeView, TableDropView, table_view, ) @@ -93,31 +61,24 @@ from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( - PaginatedResources, PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, async_call_with_supported_arguments, await_me_maybe, - baseconv, call_with_supported_arguments, - detect_json1, display_actor, escape_css_string, escape_sqlite, find_spatialite, format_bytes, module_from_path, - move_plugins_and_allow, - move_table_config, parse_metadata, resolve_env_secrets, resolve_routes, tilde_decode, - tilde_encode, to_css_class, urlsafe_components, - redact_keys, row_sql_params_pks, ) from .utils.asgi import ( @@ -135,7 +96,6 @@ from .utils.asgi import ( asgi_send_file, asgi_send_redirect, ) -from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -145,44 +105,11 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .resources import DatabaseResource, TableResource - app_root = Path(__file__).parent.parent - -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) - - -class _DatasetteClientContext: - """Context manager to mark code as executing within a datasette.client request.""" - - def __enter__(self): - self.token = _in_datasette_client.set(True) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - _in_datasette_client.reset(self.token) - return False - - -@dataclasses.dataclass -class PermissionCheck: - """Represents a logged permission check for debugging purposes.""" - - when: str - actor: Dict[str, Any] | None - action: str - parent: str | None - child: str | None - result: bool - - # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 SQLITE_LIMIT_ATTACHED = 10 -INTERNAL_DB_NAME = "__INTERNAL__" - Setting = collections.namedtuple("Setting", ("name", "default", "help")) SETTINGS = ( Setting("default_page_size", 100, "Default page size for the table view"), @@ -287,9 +214,6 @@ FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() -ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) - - async def favicon(request, send): await asgi_send_file( send, @@ -305,13 +229,6 @@ ResolvedRow = collections.namedtuple( ) -def _to_string(value): - if isinstance(value, str): - return value - else: - return json.dumps(value, default=str) - - class Datasette: # Message constants: INFO = 1 @@ -340,10 +257,8 @@ class Datasette: crossdb=False, nolock=False, internal=None, - default_deny=False, ): self._startup_invoked = False - self._closed = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -372,8 +287,7 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() - self.actions = {} # .invoke_startup() will populate this - self._column_types = {} # .invoke_startup() will populate this + self.permissions = {} # .invoke_startup() will populate this try: self._refresh_schemas_lock = asyncio.Lock() except RuntimeError as rex: @@ -398,10 +312,10 @@ class Datasette: self.internal_db_created = False if internal is None: - self._internal_database = Database(self, is_temp_disk=True) + self._internal_database = Database(self, memory_name=secrets.token_hex()) else: self._internal_database = Database(self, path=internal, mode="rwc") - self._internal_database.name = INTERNAL_DB_NAME + self._internal_database.name = "__INTERNAL__" self.cache_headers = cache_headers self.cors = cors @@ -426,13 +340,6 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - # Move any "plugins" and "allow" settings from metadata to config - updates them in place - metadata = metadata or {} - config = config or {} - metadata, config = move_plugins_and_allow(metadata, config) - # Now migrate any known table configuration settings over as well - metadata, config = move_table_config(metadata, config) - self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: @@ -457,38 +364,11 @@ class Datasette: config = config or {} config_settings = config.get("settings") or {} - # Validate settings from config file - for key, value in config_settings.items(): + # validate "settings" keys in datasette.json + for key in config_settings: if key not in DEFAULT_SETTINGS: - raise StartupError(f"Invalid setting '{key}' in config file") - # Validate type matches expected type from DEFAULT_SETTINGS - if value is not None: # Allow None/null values - expected_type = type(DEFAULT_SETTINGS[key]) - actual_type = type(value) - if actual_type != expected_type: - raise StartupError( - f"Setting '{key}' in config file has incorrect type. " - f"Expected {expected_type.__name__}, got {actual_type.__name__}. " - f"Value: {value!r}. " - f"Hint: In YAML/JSON config files, remove quotes from boolean and integer values." - ) + raise StartupError("Invalid setting '{}' in datasette.json".format(key)) - # Validate settings from constructor parameter - if settings: - for key, value in settings.items(): - if key not in DEFAULT_SETTINGS: - raise StartupError(f"Invalid setting '{key}' in settings parameter") - if value is not None: - expected_type = type(DEFAULT_SETTINGS[key]) - actual_type = type(value) - if actual_type != expected_type: - raise StartupError( - f"Setting '{key}' in settings parameter has incorrect type. " - f"Expected {expected_type.__name__}, got {actual_type.__name__}. " - f"Value: {value!r}" - ) - - self.config = config # CLI settings should overwrite datasette.json settings self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions @@ -536,87 +416,36 @@ class Datasette: ), ] ) - environment = Environment( + self.jinja_env = Environment( loader=template_loader, autoescape=True, enable_async=True, # undefined=StrictUndefined, ) - environment.filters["escape_css_string"] = escape_css_string - environment.filters["quote_plus"] = urllib.parse.quote_plus - self._jinja_env = environment - environment.filters["escape_sqlite"] = escape_sqlite - environment.filters["to_css_class"] = to_css_class + self.jinja_env.filters["escape_css_string"] = escape_css_string + self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus + self.jinja_env.filters["escape_sqlite"] = escape_sqlite + self.jinja_env.filters["to_css_class"] = to_css_class self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) - self.root_enabled = False - self.default_deny = default_deny self.client = DatasetteClient(self) - async def apply_metadata_json(self): - # Apply any metadata entries from metadata.json to the internal tables - # step 1: top-level metadata - for key in self._metadata_local or {}: - if key == "databases": - continue - value = self._metadata_local[key] - await self.set_instance_metadata(key, _to_string(value)) - - # step 2: database-level metadata - for dbname, db in self._metadata_local.get("databases", {}).items(): - for key, value in db.items(): - if key in ("tables", "queries"): - continue - await self.set_database_metadata(dbname, key, _to_string(value)) - - # step 3: table-level metadata - for tablename, table in db.get("tables", {}).items(): - for key, value in table.items(): - if key == "columns": - continue - await self.set_resource_metadata( - dbname, tablename, key, _to_string(value) - ) - - # step 4: column-level metadata (only descriptions in metadata.json) - for columnname, column_description in table.get("columns", {}).items(): - await self.set_column_metadata( - dbname, tablename, columnname, "description", column_description - ) - - # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log - # a warning to user that they should delete their metadata.json file - - async def _save_queries_from_config(self): - await stored_queries.save_queries_from_config(self) - - def get_jinja_environment(self, request: Request = None) -> Environment: - environment = self._jinja_env - if request: - for environment in pm.hook.jinja2_environment_from_request( - datasette=self, request=request, env=environment - ): - pass - return environment - - def get_action(self, name_or_abbr: str): + def get_permission(self, name_or_abbr: str) -> "Permission": """ - Returns an Action object for the given name or abbreviation. Returns None if not found. + Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. """ - if name_or_abbr in self.actions: - return self.actions[name_or_abbr] + if name_or_abbr in self.permissions: + return self.permissions[name_or_abbr] # Try abbreviation - for action in self.actions.values(): - if action.abbr == name_or_abbr: - return action - return None + for permission in self.permissions.values(): + if permission.abbr == name_or_abbr: + return permission + raise KeyError( + "No permission found with name or abbreviation {}".format(name_or_abbr) + ) async def refresh_schemas(self): - # Throttle schema refreshes to at most once per second - if time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0: - return - self._last_schema_refresh = time.monotonic() if self._refresh_schemas_lock.locked(): return async with self._refresh_schemas_lock: @@ -626,7 +455,6 @@ class Datasette: internal_db = self.get_internal_database() if not self.internal_db_created: await init_internal_db(internal_db) - await self.apply_metadata_json() self.internal_db_created = True current_schema_versions = { row["database_name"]: row["schema_version"] @@ -634,36 +462,6 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } - catalog_table_names = ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - "catalog_databases", - ) - # Delete stale entries for databases that are no longer attached - catalog_database_names = set(current_schema_versions.keys()) - for table in catalog_table_names[:-1]: - catalog_database_names.update( - row["database_name"] - for row in await internal_db.execute( - "select distinct database_name from {}".format(table) - ) - if row["database_name"] is not None - ) - stale_databases = catalog_database_names - set(self.databases.keys()) - if stale_databases: - - def delete_stale_database_catalog(conn): - for stale_db_name in stale_databases: - for table in catalog_table_names: - conn.execute( - "DELETE FROM {} WHERE database_name = ?".format(table), - [stale_db_name], - ) - - await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it @@ -678,7 +476,9 @@ class Datasette: """ INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) VALUES {} - """.format(placeholders), + """.format( + placeholders + ), values, ) await populate_schema_tables(internal_db, db) @@ -687,73 +487,32 @@ class Datasette: def urls(self): return Urls(self) - @property - def pm(self): - """ - Return the global plugin manager instance. - - This provides access to the pluggy PluginManager that manages all - Datasette plugins and hooks. Use datasette.pm.hook.hook_name() to - call plugin hooks. - """ - return pm - async def invoke_startup(self): # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # Register event classes - event_classes = [] - for hook in pm.hook.register_events(datasette=self): - extra_classes = await await_me_maybe(hook) - if extra_classes: - event_classes.extend(extra_classes) - self.event_classes = tuple(event_classes) - - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): + # Register permissions, but watch out for duplicate name/abbr + names = {} + abbrs = {} + for hook in pm.hook.register_permissions(datasette=self): if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): + for p in hook: + if p.name in names and p != names[p.name]: raise StartupError( - "Duplicate action name: {}".format(action.name) + "Duplicate permission name: {}".format(p.name) ) - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): + if p.abbr and p.abbr in abbrs and p != abbrs[p.abbr]: raise StartupError( - "Duplicate action abbr: {}".format(action.abbr) + "Duplicate permission abbr: {}".format(p.abbr) ) - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action - - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError(f"Duplicate column type name: {ct_cls.name}") - self._column_types[ct_cls.name] = ct_cls - + names[p.name] = p + if p.abbr: + abbrs[p.abbr] = p + self.permissions[p.name] = p for hook in pm.hook.prepare_jinja2_environment( - env=self._jinja_env, datasette=self + env=self.jinja_env, datasette=self ): await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) self._startup_invoked = True @@ -764,78 +523,44 @@ class Datasette: def unsign(self, signed, namespace="default"): return URLSafeSerializer(self._secret, namespace).loads(signed) - def in_client(self) -> bool: - """Check if the current code is executing within a datasette.client request. - - Returns: - bool: True if currently executing within a datasette.client request, False otherwise. - """ - return _in_datasette_client.get() - - def _token_handlers(self): - """Collect all registered token handlers from plugins.""" - from datasette.tokens import TokenHandler - - handlers = [] - for result in pm.hook.register_token_handler(datasette=self): - if isinstance(result, TokenHandler): - handlers.append(result) - elif isinstance(result, list): - handlers.extend(h for h in result if isinstance(h, TokenHandler)) - return handlers - - async def create_token( + def create_token( self, actor_id: str, *, - expires_after: int | None = None, - restrictions: "TokenRestrictions | None" = None, - handler: str | None = None, - ) -> str: - """ - Create an API token for the given actor. + expires_after: Optional[int] = None, + restrict_all: Optional[Iterable[str]] = None, + restrict_database: Optional[Dict[str, Iterable[str]]] = None, + restrict_resource: Optional[Dict[str, Dict[str, Iterable[str]]]] = None, + ): + token = {"a": actor_id, "t": int(time.time())} + if expires_after: + token["d"] = expires_after - Uses the first registered token handler by default, or a specific - handler if ``handler`` is provided (matched by handler name). + def abbreviate_action(action): + # rename to abbr if possible + permission = self.permissions.get(action) + if not permission: + return action + return permission.abbr or action - Pass a :class:`TokenRestrictions` to limit which actions the token - can perform. - """ - handlers = self._token_handlers() - if not handlers: - raise RuntimeError("No token handlers are registered") - - if handler is not None: - matched = [h for h in handlers if h.name == handler] - if not matched: - available = [h.name for h in handlers] - raise ValueError( - f"Token handler {handler!r} not found. " - f"Available handlers: {available}" - ) - chosen = matched[0] - else: - chosen = handlers[0] - - return await chosen.create_token( - self, - actor_id, - expires_after=expires_after, - restrictions=restrictions, - ) - - async def verify_token(self, token: str) -> dict | None: - """ - Verify an API token by trying all registered token handlers. - - Returns an actor dict from the first handler that recognizes the - token, or None if no handler accepts it. - """ - for token_handler in self._token_handlers(): - result = await token_handler.verify_token(self, token) - if result is not None: - return result - return None + if expires_after: + token["d"] = expires_after + if restrict_all or restrict_database or restrict_resource: + token["_r"] = {} + if restrict_all: + token["_r"]["a"] = [abbreviate_action(a) for a in restrict_all] + if restrict_database: + token["_r"]["d"] = {} + for database, actions in restrict_database.items(): + token["_r"]["d"][database] = [abbreviate_action(a) for a in actions] + if restrict_resource: + token["_r"]["r"] = {} + for database, resources in restrict_resource.items(): + for resource, actions in resources.items(): + token["_r"]["r"].setdefault(database, {})[resource] = [ + abbreviate_action(a) for a in actions + ] + return "dstok_{}".format(self.sign(token, namespace="token")) def get_database(self, name=None, route=None): if route is not None: @@ -866,44 +591,14 @@ class Datasette: self.databases = new_databases return db - def add_memory_database(self, memory_name, name=None, route=None): - return self.add_database( - Database(self, memory_name=memory_name), name=name, route=route - ) + def add_memory_database(self, memory_name): + return self.add_database(Database(self, memory_name=memory_name)) def remove_database(self, name): - self.get_database(name).close() new_databases = self.databases.copy() new_databases.pop(name) self.databases = new_databases - def close(self): - """Release all resources held by this Datasette instance. - - Closes every attached Database (including the internal database), - shuts down the executor, and unlinks the temporary file used for - the internal database if one was created. Idempotent and one-way. - """ - if self._closed: - return - self._closed = True - first_exception = None - dbs = list(self.databases.values()) + [self._internal_database] - for db in dbs: - try: - db.close() - except Exception as e: - if first_exception is None: - first_exception = e - if self.executor is not None: - try: - self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: - if first_exception is None: - first_exception = e - if first_exception is not None: - raise first_exception - def setting(self, key): return self._settings.get(key, None) @@ -922,520 +617,122 @@ class Datasette: orig[key] = upd_value return orig - async def get_instance_metadata(self): - rows = await self.get_internal_database().execute(""" - SELECT - key, - value - FROM metadata_instance - """) - return dict(rows) - - async def get_database_metadata(self, database_name: str): - rows = await self.get_internal_database().execute( - """ - SELECT - key, - value - FROM metadata_databases - WHERE database_name = ? - """, - [database_name], - ) - return dict(rows) - - async def get_resource_metadata(self, database_name: str, resource_name: str): - rows = await self.get_internal_database().execute( - """ - SELECT - key, - value - FROM metadata_resources - WHERE database_name = ? - AND resource_name = ? - """, - [database_name, resource_name], - ) - return dict(rows) - - async def get_column_metadata( - self, database_name: str, resource_name: str, column_name: str - ): - rows = await self.get_internal_database().execute( - """ - SELECT - key, - value - FROM metadata_columns - WHERE database_name = ? - AND resource_name = ? - AND column_name = ? - """, - [database_name, resource_name, column_name], - ) - return dict(rows) - - async def set_instance_metadata(self, key: str, value: str): - # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) - await self.get_internal_database().execute_write( - """ - INSERT INTO metadata_instance(key, value) - VALUES(?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - """, - [key, value], - ) - - async def set_database_metadata(self, database_name: str, key: str, value: str): - # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) - await self.get_internal_database().execute_write( - """ - INSERT INTO metadata_databases(database_name, key, value) - VALUES(?, ?, ?) - ON CONFLICT(database_name, key) DO UPDATE SET value = excluded.value; - """, - [database_name, key, value], - ) - - async def set_resource_metadata( - self, database_name: str, resource_name: str, key: str, value: str - ): - # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) - await self.get_internal_database().execute_write( - """ - INSERT INTO metadata_resources(database_name, resource_name, key, value) - VALUES(?, ?, ?, ?) - ON CONFLICT(database_name, resource_name, key) DO UPDATE SET value = excluded.value; - """, - [database_name, resource_name, key, value], - ) - - async def set_column_metadata( - self, - database_name: str, - resource_name: str, - column_name: str, - key: str, - value: str, - ): - # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) - await self.get_internal_database().execute_write( - """ - INSERT INTO metadata_columns(database_name, resource_name, column_name, key, value) - VALUES(?, ?, ?, ?, ?) - ON CONFLICT(database_name, resource_name, column_name, key) DO UPDATE SET value = excluded.value; - """, - [database_name, resource_name, column_name, key, value], - ) - - @staticmethod - def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: - return stored_queries.query_row_to_stored_query(row) - - @staticmethod - def _query_options_json(options): - return stored_queries.query_options_json(options) - - async def add_query( - self, - database: str, - name: str, - sql: str, - *, - title: str | None = None, - description: str | None = None, - description_html: str | None = None, - hide_sql: bool = False, - fragment: str | None = None, - parameters: Iterable[str] | None = None, - is_write: bool = False, - is_private: bool = False, - is_trusted: bool = False, - source: str = "plugin", - owner_id: str | None = None, - on_success_message: str | None = None, - on_success_message_sql: str | None = None, - on_success_redirect: str | None = None, - on_error_message: str | None = None, - on_error_redirect: str | None = None, - replace: bool = True, - ) -> None: - return await stored_queries.add_query( - self, - database, - name, - sql, - title=title, - description=description, - description_html=description_html, - hide_sql=hide_sql, - fragment=fragment, - parameters=parameters, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - on_success_message=on_success_message, - on_success_message_sql=on_success_message_sql, - on_success_redirect=on_success_redirect, - on_error_message=on_error_message, - on_error_redirect=on_error_redirect, - replace=replace, - ) - - async def update_query( - self, - database: str, - name: str, - *, - sql=stored_queries.UNCHANGED, - title=stored_queries.UNCHANGED, - description=stored_queries.UNCHANGED, - description_html=stored_queries.UNCHANGED, - hide_sql=stored_queries.UNCHANGED, - fragment=stored_queries.UNCHANGED, - parameters=stored_queries.UNCHANGED, - is_write=stored_queries.UNCHANGED, - is_private=stored_queries.UNCHANGED, - is_trusted=stored_queries.UNCHANGED, - source=stored_queries.UNCHANGED, - owner_id=stored_queries.UNCHANGED, - on_success_message=stored_queries.UNCHANGED, - on_success_message_sql=stored_queries.UNCHANGED, - on_success_redirect=stored_queries.UNCHANGED, - on_error_message=stored_queries.UNCHANGED, - on_error_redirect=stored_queries.UNCHANGED, - ) -> None: - return await stored_queries.update_query( - self, - database, - name, - sql=sql, - title=title, - description=description, - description_html=description_html, - hide_sql=hide_sql, - fragment=fragment, - parameters=parameters, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - on_success_message=on_success_message, - on_success_message_sql=on_success_message_sql, - on_success_redirect=on_success_redirect, - on_error_message=on_error_message, - on_error_redirect=on_error_redirect, - ) - - async def remove_query( - self, database: str, name: str, source: str | None = None - ) -> None: - return await stored_queries.remove_query(self, database, name, source=source) - - async def get_query( - self, database: str, name: str - ) -> stored_queries.StoredQuery | None: - return await stored_queries.get_query(self, database, name) - - async def count_queries( - self, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - ) -> int: - return await stored_queries.count_queries( - self, - database, - actor=actor, - q=q, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - ) - - async def list_queries( - self, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - limit: int = 50, - cursor: str | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - include_private: bool = False, - ) -> stored_queries.StoredQueryPage: - return await stored_queries.list_queries( - self, - database, - actor=actor, - limit=limit, - cursor=cursor, - q=q, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - include_private=include_private, - ) - - async def ensure_query_write_permissions( - self, database, sql, *, actor=None, params=None, analysis=None - ): - return await stored_queries.ensure_query_write_permissions( - self, database, sql, actor=actor, params=params, analysis=analysis - ) - - # Column types API - - async def _get_resource_column_details(self, database: str, resource: str): - db = self.databases.get(database) - if db is None: - return {} - try: - return { - column.name: column - for column in await db.table_column_details(resource) - } - except sqlite3.OperationalError: - return {} - - @staticmethod - def _column_type_is_applicable(ct_cls, column_detail) -> bool: - sqlite_types = getattr(ct_cls, "sqlite_types", None) - if sqlite_types is None: - return True - if column_detail is None: - return False - actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type) - return actual_sqlite_type in sqlite_types - - async def _validate_column_type_assignment( - self, database: str, resource: str, column: str, ct_cls - ) -> None: - sqlite_types = getattr(ct_cls, "sqlite_types", None) - if sqlite_types is None: - return - - column_detail = ( - await self._get_resource_column_details(database, resource) - ).get(column) - if column_detail is None: - return - - actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type) - if actual_sqlite_type in sqlite_types: - return - - allowed = ", ".join(sqlite_type.value for sqlite_type in sqlite_types) - actual = ( - actual_sqlite_type.value - if actual_sqlite_type is not None - else "unrecognized {!r}".format(column_detail.type) - ) - raise ValueError( - "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " - "has SQLite type {}".format( - ct_cls.name, - allowed, - database, - resource, - column, - actual, - ) - ) - - async def _apply_column_types_config(self): - """Load column_types from datasette.json config into the internal DB.""" - import logging - - for db_name, db_conf in (self.config or {}).get("databases", {}).items(): - for table_name, table_conf in db_conf.get("tables", {}).items(): - for col_name, ct in table_conf.get("column_types", {}).items(): - if isinstance(ct, str): - col_type, config = ct, None - else: - col_type = ct["type"] - config = ct.get("config") - if col_type not in self._column_types: - logging.warning( - "column_types config references unknown type %r " - "for %s.%s.%s", - col_type, - db_name, - table_name, - col_name, - ) - try: - await self.set_column_type( - db_name, table_name, col_name, col_type, config - ) - except ValueError as ex: - logging.warning(str(ex)) - - async def get_column_type(self, database: str, resource: str, column: str): + def metadata(self, key=None, database=None, table=None, fallback=True): """ - Return a ColumnType instance (with config baked in) for a specific - column, or None if no column type is assigned. + Looks up metadata, cascading backwards from specified level. + Returns None if metadata value is not found. """ - row = await self.get_internal_database().execute( - "SELECT column_type, config FROM column_types " - "WHERE database_name = ? AND resource_name = ? AND column_name = ?", - [database, resource, column], - ) - rows = row.rows - if not rows: + assert not ( + database is None and table is not None + ), "Cannot call metadata() with table= specified but not database=" + metadata = {} + + for hook_dbs in pm.hook.get_metadata( + datasette=self, key=key, database=database, table=table + ): + metadata = self._metadata_recursive_update(metadata, hook_dbs) + + # security precaution!! don't allow anything in the local config + # to be overwritten. this is a temporary measure, not sure if this + # is a good idea long term or maybe if it should just be a concern + # of the plugin's implemtnation + metadata = self._metadata_recursive_update(metadata, self._metadata_local) + + databases = metadata.get("databases") or {} + + search_list = [] + if database is not None: + search_list.append(databases.get(database) or {}) + if table is not None: + table_metadata = ((databases.get(database) or {}).get("tables") or {}).get( + table + ) or {} + search_list.insert(0, table_metadata) + + search_list.append(metadata) + if not fallback: + # No fallback allowed, so just use the first one in the list + search_list = search_list[:1] + if key is not None: + for item in search_list: + if key in item: + return item[key] return None - ct_name, config = rows[0] - ct_cls = self._column_types.get(ct_name) - if ct_cls is None: - return None - column_detail = ( - await self._get_resource_column_details(database, resource) - ).get(column) - if not self._column_type_is_applicable(ct_cls, column_detail): - return None - return ct_cls(config=json.loads(config) if config else None) + else: + # Return the merged list + m = {} + for item in search_list: + m.update(item) + return m - async def get_column_types(self, database: str, resource: str) -> dict: - """ - Return {column_name: ColumnType instance (with config)} - for all columns with assigned types on the given resource. - """ - rows = await self.get_internal_database().execute( - "SELECT column_name, column_type, config FROM column_types " - "WHERE database_name = ? AND resource_name = ?", - [database, resource], - ) - column_details = await self._get_resource_column_details(database, resource) - result = {} - for row in rows.rows: - col_name, ct_name, config = row - ct_cls = self._column_types.get(ct_name) - if ct_cls is not None and self._column_type_is_applicable( - ct_cls, column_details.get(col_name) - ): - result[col_name] = ct_cls(config=json.loads(config) if config else None) - return result - - async def set_column_type( - self, - database: str, - resource: str, - column: str, - column_type: str, - config: dict = None, - ) -> None: - """Assign a column type. Overwrites any existing assignment.""" - ct_cls = self._column_types.get(column_type) - if ct_cls is not None: - await self._validate_column_type_assignment( - database, resource, column, ct_cls - ) - await self.get_internal_database().execute_write( - """INSERT OR REPLACE INTO column_types - (database_name, resource_name, column_name, column_type, config) - VALUES (?, ?, ?, ?, ?)""", - [ - database, - resource, - column, - column_type, - json.dumps(config) if config else None, - ], - ) - - async def remove_column_type( - self, database: str, resource: str, column: str - ) -> None: - """Remove a column type assignment.""" - await self.get_internal_database().execute_write( - "DELETE FROM column_types " - "WHERE database_name = ? AND resource_name = ? AND column_name = ?", - [database, resource, column], - ) + @property + def _metadata(self): + return self.metadata() def get_internal_database(self): return self._internal_database def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" - if database is None and table is None: - config = self._plugin_config_top(plugin_name) - else: - config = self._plugin_config_nested(plugin_name, database, table, fallback) - - return resolve_env_secrets(config, os.environ) - - def _plugin_config_top(self, plugin_name): - """Returns any top-level plugin configuration for the specified plugin.""" - return ((self.config or {}).get("plugins") or {}).get(plugin_name) - - def _plugin_config_nested(self, plugin_name, database, table=None, fallback=True): - """Returns any database or table-level plugin configuration for the specified plugin.""" - db_config = ((self.config or {}).get("databases") or {}).get(database) - - # if there's no db-level configuration, then return early, falling back to top-level if needed - if not db_config: - return self._plugin_config_top(plugin_name) if fallback else None - - db_plugin_config = (db_config.get("plugins") or {}).get(plugin_name) - - if table: - table_plugin_config = ( - ((db_config.get("tables") or {}).get(table) or {}).get("plugins") or {} - ).get(plugin_name) - - # fallback to db_config or top-level config, in that order, if needed - if table_plugin_config is None and fallback: - return db_plugin_config or self._plugin_config_top(plugin_name) - - return table_plugin_config - - # fallback to top-level if needed - if db_plugin_config is None and fallback: - self._plugin_config_top(plugin_name) - - return db_plugin_config - - def static_hash(self, filename): - if not hasattr(self, "_static_hashes"): - self._static_hashes = {} - path = os.path.join(str(app_root), "datasette/static", filename) - signature = (os.path.getmtime(path), os.path.getsize(path)) - cached = self._static_hashes.get(filename) - if cached and cached["signature"] == signature: - return cached["hash"] - with open(path) as fp: - static_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[:6] - self._static_hashes[filename] = { - "signature": signature, - "hash": static_hash, - } - return static_hash + plugins = self.metadata( + "plugins", database=database, table=table, fallback=fallback + ) + if plugins is None: + return None + plugin_config = plugins.get(plugin_name) + # Resolve any $file and $env keys + plugin_config = resolve_env_secrets(plugin_config, os.environ) + return plugin_config def app_css_hash(self): - return self.static_hash("app.css") + if not hasattr(self, "_app_css_hash"): + with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: + self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ + :6 + ] + return self._app_css_hash + + async def get_canned_queries(self, database_name, actor): + queries = self.metadata("queries", database=database_name, fallback=False) or {} + for more_queries in pm.hook.canned_queries( + datasette=self, + database=database_name, + actor=actor, + ): + more_queries = await await_me_maybe(more_queries) + queries.update(more_queries or {}) + # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}} + for key in queries: + if not isinstance(queries[key], dict): + queries[key] = {"sql": queries[key]} + # Also make sure "name" is available: + queries[key]["name"] = key + return queries + + async def get_canned_query(self, database_name, query_name, actor): + queries = await self.get_canned_queries(database_name, actor) + query = queries.get(query_name) + if query: + return query + + def update_with_inherited_metadata(self, metadata): + # Fills in source/license with defaults, if available + metadata.update( + { + "source": metadata.get("source") or self.metadata("source"), + "source_url": metadata.get("source_url") or self.metadata("source_url"), + "license": metadata.get("license") or self.metadata("license"), + "license_url": metadata.get("license_url") + or self.metadata("license_url"), + "about": metadata.get("about") or self.metadata("about"), + "about_url": metadata.get("about_url") or self.metadata("about_url"), + } + ) def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") - if self.sqlite_extensions and database != INTERNAL_DB_NAME: + if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: # "extension" is either a string path to the extension @@ -1448,8 +745,7 @@ class Datasette: if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member - if database != INTERNAL_DB_NAME: - pm.hook.prepare_connection(conn=conn, database=database, datasette=self) + pm.hook.prepare_connection(conn=conn, database=database, datasette=self) # If self.crossdb and this is _memory, connect the first SQLITE_LIMIT_ATTACHED databases if self.crossdb and database == "_memory": count = 0 @@ -1492,14 +788,14 @@ class Datasette: if request: actor = request.actor # Top-level link - if await self.allowed(action="view-instance", actor=actor): + if await self.permission_allowed(actor=actor, action="view-instance"): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link if database: - if await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), + if await self.permission_allowed( actor=actor, + action="view-database", + resource=database, ): crumbs.append( { @@ -1510,10 +806,10 @@ class Datasette: # Table link if table: assert database, "table= requires database=" - if await self.allowed( - action="view-table", - resource=TableResource(database=database, table=table), + if await self.permission_allowed( actor=actor, + action="view-table", + resource=(database, table), ): crumbs.append( { @@ -1524,8 +820,8 @@ class Datasette: return crumbs async def actors_from_ids( - self, actor_ids: Iterable[str | int] - ) -> Dict[int | str, Dict]: + self, actor_ids: Iterable[Union[str, int]] + ) -> Dict[Union[id, str], Dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1533,362 +829,103 @@ class Datasette: result = await await_me_maybe(result) return result - async def track_event(self, event: Event): - assert isinstance(event, self.event_classes), "Invalid event type: {}".format( - type(event) + async def permission_allowed( + self, actor, action, resource=None, default=DEFAULT_NOT_SET + ): + """Check permissions using the permissions_allowed plugin hook""" + result = None + # Use default from registered permission, if available + if default is DEFAULT_NOT_SET and action in self.permissions: + default = self.permissions[action].default + for check in pm.hook.permission_allowed( + datasette=self, + actor=actor, + action=action, + resource=resource, + ): + check = await await_me_maybe(check) + if check is not None: + result = check + used_default = False + if result is None: + result = default + used_default = True + self._permission_checks.append( + { + "when": datetime.datetime.utcnow().isoformat(), + "actor": actor, + "action": action, + "resource": resource, + "used_default": used_default, + "result": result, + } ) - for hook in pm.hook.track_event(datasette=self, event=event): - await await_me_maybe(hook) + return result - def resource_for_action(self, action: str, parent: str | None, child: str | None): + async def ensure_permissions( + self, + actor: dict, + permissions: Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]], + ): """ - Create a Resource instance for the given action with parent/child values. + permissions is a list of (action, resource) tuples or 'action' strings - Looks up the action's resource_class and instantiates it with the - provided parent and child identifiers. - - Args: - action: The action name (e.g., "view-table", "view-query") - parent: The parent resource identifier (e.g., database name) - child: The child resource identifier (e.g., table/query name) - - Returns: - A Resource instance of the appropriate subclass - - Raises: - ValueError: If the action is unknown + Raises datasette.Forbidden() if any of the checks fail """ - from datasette.permissions import Resource - - action_obj = self.actions.get(action) - if not action_obj: - raise ValueError(f"Unknown action: {action}") - - resource_class = action_obj.resource_class - instance = object.__new__(resource_class) - Resource.__init__(instance, parent=parent, child=child) - return instance + assert actor is None or isinstance(actor, dict), "actor must be None or a dict" + for permission in permissions: + if isinstance(permission, str): + action = permission + resource = None + elif isinstance(permission, (tuple, list)) and len(permission) == 2: + action, resource = permission + else: + assert ( + False + ), "permission should be string or tuple of two items: {}".format( + repr(permission) + ) + ok = await self.permission_allowed( + actor, + action, + resource=resource, + default=None, + ) + if ok is not None: + if ok: + return + else: + raise Forbidden(action) async def check_visibility( self, actor: dict, - action: str, - resource: "Resource" | None = None, + action: Optional[str] = None, + resource: Optional[Union[str, Tuple[str, str]]] = None, + permissions: Optional[ + Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]] + ] = None, ): - """ - Check if actor can see a resource and if it's private. - - Returns (visible, private) tuple: - - visible: bool - can the actor see it? - - private: bool - if visible, can anonymous users NOT see it? - """ - from datasette.permissions import Resource - - # Validate that resource is a Resource object or None - if resource is not None and not isinstance(resource, Resource): - raise TypeError("resource must be a Resource subclass instance or None.") - - # Check if actor can see it - if not await self.allowed(action=action, resource=resource, actor=actor): + """Returns (visible, private) - visible = can you see it, private = can others see it too""" + if permissions: + assert ( + not action and not resource + ), "Can't use action= or resource= with permissions=" + else: + permissions = [(action, resource)] + try: + await self.ensure_permissions(actor, permissions) + except Forbidden: return False, False - - # Check if anonymous user can see it (for "private" flag) - if not await self.allowed(action=action, resource=resource, actor=None): - # Actor can see it but anonymous cannot - it's private + # User can see it, but can the anonymous user see it? + try: + await self.ensure_permissions(None, permissions) + except Forbidden: + # It's visible but private return True, True - - # Both actor and anonymous can see it - it's public + # It's visible to everyone return True, False - async def allowed_resources_sql( - self, - *, - action: str, - actor: dict | None = None, - parent: str | None = None, - include_is_private: bool = False, - ) -> ResourcesSQL: - """ - Build SQL query to get all resources the actor can access for the given action. - - Args: - action: The action name (e.g., "view-table") - actor: The actor dict (or None for unauthenticated) - parent: Optional parent filter (e.g., database name) to limit results - include_is_private: If True, include is_private column showing if anonymous cannot access - - Returns a namedtuple of (query: str, params: dict) that can be executed against the internal database. - The query returns rows with (parent, child, reason) columns, plus is_private if requested. - - Example: - query, params = await datasette.allowed_resources_sql( - action="view-table", - actor=actor, - parent="mydb", - include_is_private=True - ) - result = await datasette.get_internal_database().execute(query, params) - """ - from datasette.utils.actions_sql import build_allowed_resources_sql - - action_obj = self.actions.get(action) - if not action_obj: - raise ValueError(f"Unknown action: {action}") - - sql, params = await build_allowed_resources_sql( - self, actor, action, parent=parent, include_is_private=include_is_private - ) - return ResourcesSQL(sql, params) - - async def allowed_resources( - self, - action: str, - actor: dict | None = None, - *, - parent: str | None = None, - include_is_private: bool = False, - include_reasons: bool = False, - limit: int = 100, - next: str | None = None, - ) -> PaginatedResources: - """ - Return paginated resources the actor can access for the given action. - - Uses SQL with keyset pagination to efficiently filter resources. - Returns PaginatedResources with list of Resource instances and pagination metadata. - - Args: - action: The action name (e.g., "view-table") - actor: The actor dict (or None for unauthenticated) - parent: Optional parent filter (e.g., database name) to limit results - include_is_private: If True, adds a .private attribute to each Resource - include_reasons: If True, adds a .reasons attribute with List[str] of permission reasons - limit: Maximum number of results to return (1-1000, default 100) - next: Keyset token from previous page for pagination - - Returns: - PaginatedResources with: - - resources: List of Resource objects for this page - - next: Token for next page (None if no more results) - - Example: - # Get first page of tables - page = await datasette.allowed_resources("view-table", actor, limit=50) - for table in page.resources: - print(f"{table.parent}/{table.child}") - - # Get next page - if page.next: - next_page = await datasette.allowed_resources( - "view-table", actor, limit=50, next=page.next - ) - - # With reasons for debugging - page = await datasette.allowed_resources( - "view-table", actor, include_reasons=True - ) - for table in page.resources: - print(f"{table.child}: {table.reasons}") - - # Iterate through all results with async generator - page = await datasette.allowed_resources("view-table", actor) - async for table in page.all(): - print(table.child) - """ - - action_obj = self.actions.get(action) - if not action_obj: - raise ValueError(f"Unknown action: {action}") - - # Validate and cap limit - limit = min(max(1, limit), 1000) - - # Get base SQL query - query, params = await self.allowed_resources_sql( - action=action, - actor=actor, - parent=parent, - include_is_private=include_is_private, - ) - - # Add keyset pagination WHERE clause if next token provided - if next: - try: - components = urlsafe_components(next) - if len(components) >= 2: - last_parent, last_child = components[0], components[1] - # Keyset condition: (parent > last) OR (parent = last AND child > last) - keyset_where = """ - (parent > :keyset_parent OR - (parent = :keyset_parent AND child > :keyset_child)) - """ - # Wrap original query and add keyset filter - query = f"SELECT * FROM ({query}) WHERE {keyset_where}" - params["keyset_parent"] = last_parent - params["keyset_child"] = last_child - except (ValueError, KeyError): - # Invalid token - ignore and start from beginning - pass - - # Add LIMIT (fetch limit+1 to detect if there are more results) - # Note: query from allowed_resources_sql() already includes ORDER BY parent, child - query = f"{query} LIMIT :limit" - params["limit"] = limit + 1 - - # Execute query - result = await self.get_internal_database().execute(query, params) - rows = list(result.rows) - - # Check if truncated (got more than limit rows) - truncated = len(rows) > limit - if truncated: - rows = rows[:limit] # Remove the extra row - - # Build Resource objects with optional attributes - resources = [] - for row in rows: - # row[0]=parent, row[1]=child, row[2]=reason, row[3]=is_private (if requested) - resource = self.resource_for_action(action, parent=row[0], child=row[1]) - - # Add reasons if requested - if include_reasons: - reason_json = row[2] - try: - reasons_array = ( - json.loads(reason_json) if isinstance(reason_json, str) else [] - ) - resource.reasons = [r for r in reasons_array if r is not None] - except (json.JSONDecodeError, TypeError): - resource.reasons = [reason_json] if reason_json else [] - - # Add private flag if requested - if include_is_private: - resource.private = bool(row[3]) - - resources.append(resource) - - # Generate next token if there are more results - next_token = None - if truncated and resources: - last_resource = resources[-1] - # Use tilde-encoding like table pagination - next_token = "{},{}".format( - tilde_encode(str(last_resource.parent)), - tilde_encode(str(last_resource.child)), - ) - - return PaginatedResources( - resources=resources, - next=next_token, - _datasette=self, - _action=action, - _actor=actor, - _parent=parent, - _include_is_private=include_is_private, - _include_reasons=include_reasons, - _limit=limit, - ) - - async def allowed( - self, - *, - action: str, - resource: "Resource" = None, - actor: dict | None = None, - ) -> bool: - """ - Check if actor can perform action on specific resource. - - Uses SQL to check permission for a single resource without fetching all resources. - This is efficient - it does NOT call allowed_resources() and check membership. - - For global actions, resource should be None (or omitted). - - Example: - from datasette.resources import TableResource - can_view = await datasette.allowed( - action="view-table", - resource=TableResource(database="analytics", table="users"), - actor=actor - ) - - # For global actions, resource can be omitted: - can_debug = await datasette.allowed(action="permissions-debug", actor=actor) - """ - from datasette.utils.actions_sql import check_permission_for_resource - - # For global actions, resource remains None - - # Check if this action has also_requires - if so, check that action first - action_obj = self.actions.get(action) - if action_obj and action_obj.also_requires: - # Must have the required action first - if not await self.allowed( - action=action_obj.also_requires, - resource=resource, - actor=actor, - ): - return False - - # For global actions, resource is None - parent = resource.parent if resource else None - child = resource.child if resource else None - - result = await check_permission_for_resource( - datasette=self, - actor=actor, - action=action, - parent=parent, - child=child, - ) - - # Log the permission check for debugging - self._permission_checks.append( - PermissionCheck( - when=datetime.datetime.now(datetime.timezone.utc).isoformat(), - actor=actor, - action=action, - parent=parent, - child=child, - result=result, - ) - ) - - return result - - async def ensure_permission( - self, - *, - action: str, - resource: "Resource" = None, - actor: dict | None = None, - ): - """ - Check if actor can perform action on resource, raising Forbidden if not. - - This is a convenience wrapper around allowed() that raises Forbidden - instead of returning False. Use this when you want to enforce a permission - check and halt execution if it fails. - - Example: - from datasette.resources import TableResource - - # Will raise Forbidden if actor cannot view the table - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database="analytics", table="users"), - actor=request.actor - ) - - # For instance-level actions, resource can be omitted: - await datasette.ensure_permission( - action="permissions-debug", - actor=request.actor - ) - """ - if not await self.allowed(action=action, resource=resource, actor=actor): - raise Forbidden(action) - async def execute( self, db_name, @@ -1923,14 +960,15 @@ class Datasette: except IndexError: return {} # Ensure user has permission to view the referenced table - from datasette.resources import TableResource - other_table = fk["other_table"] other_column = fk["other_column"] visible, _ = await self.check_visibility( actor, - action="view-table", - resource=TableResource(database=database, table=other_table), + permissions=[ + ("view-table", (database, other_table)), + ("view-database", database), + "view-instance", + ], ) if not visible: return {} @@ -1963,6 +1001,11 @@ class Datasette: url = "https://" + url[len("http://") :] return url + def _register_custom_units(self): + """Register any custom units defined in the metadata.json with Pint""" + for unit in self.metadata("custom_units") or []: + ureg.define(unit) + def _connected_databases(self): return [ { @@ -1981,8 +1024,9 @@ class Datasette: conn = sqlite3.connect(":memory:") self._prepare_connection(conn, "_memory") sqlite_version = conn.execute("select sqlite_version()").fetchone()[0] - sqlite_extensions = {"json1": detect_json1(conn)} + sqlite_extensions = {} for extension, testsql, hasversion in ( + ("json1", "SELECT json('{}')", False), ("spatialite", "SELECT spatialite_version()", True), ): try: @@ -2046,11 +1090,10 @@ class Datasette: if using_pysqlite3: for package in ("pysqlite3", "pysqlite3-binary"): try: - info["pysqlite3"] = importlib.metadata.version(package) + info["pysqlite3"] = pkg_resources.get_distribution(package).version break - except importlib.metadata.PackageNotFoundError: + except pkg_resources.DistributionNotFound: pass - conn.close() return info def _plugins(self, request=None, all=False): @@ -2096,27 +1139,10 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - def _actions(self): - return [ - { - "name": action.name, - "abbr": action.abbr, - "description": action.description, - "takes_parent": action.takes_parent, - "takes_child": action.takes_child, - "resource_class": ( - action.resource_class.__name__ if action.resource_class else None - ), - "also_requires": action.also_requires, - } - for action in sorted(self.actions.values(), key=lambda a: a.name) - ] - - async def table_config(self, database: str, table: str) -> dict: - """Return dictionary of configuration for specified table""" + def table_metadata(self, database, table): + """Fetch table-specific metadata.""" return ( - (self.config or {}) - .get("databases", {}) + (self.metadata("databases") or {}) .get(database, {}) .get("tables", {}) .get(table, {}) @@ -2145,10 +1171,10 @@ class Datasette: async def render_template( self, - templates: List[str] | str | Template, - context: Dict[str, Any] | Context | None = None, - request: Request | None = None, - view_name: str | None = None, + templates: Union[List[str], str, Template], + context: Optional[Union[Dict[str, Any], Context]] = None, + request: Optional[Request] = None, + view_name: Optional[str] = None, ): if not self._startup_invoked: raise Exception("render_template() called before await ds.invoke_startup()") @@ -2158,7 +1184,7 @@ class Datasette: else: if isinstance(templates, str): templates = [templates] - template = self.get_jinja_environment(request).select_template(templates) + template = self.jinja_env.select_template(templates) if dataclasses.is_dataclass(context): context = dataclasses.asdict(context) body_scripts = [] @@ -2234,11 +1260,7 @@ class Datasette: "extra_js_urls", template, context, request, view_name ), "base_url": self.setting("base_url"), - "csrftoken": ( - request.scope["csrftoken"] - if request and "csrftoken" in request.scope - else lambda: "" - ), + "csrftoken": request.scope["csrftoken"] if request else lambda: "", "datasette_version": __version__, }, **extra_template_vars, @@ -2250,18 +1272,6 @@ class Datasette: return await template.render_async(template_context) - def set_actor_cookie( - self, response: Response, actor: dict, expire_after: int | None = None - ): - data = {"a": actor} - if expire_after: - expires_at = int(time.time()) + (24 * 60 * 60) - data["e"] = baseconv.base62.encode(expires_at) - response.set_cookie("ds_actor", self.sign(data, "actor")) - - def delete_actor_cookie(self, response: Response): - response.set_cookie("ds_actor", "", expires=0, max_age=0) - async def _asset_urls(self, key, template, context, request, view_name): # Flatten list-of-lists from plugins: seen_urls = set() @@ -2277,7 +1287,7 @@ class Datasette: ): hook = await await_me_maybe(hook) collected.extend(hook) - collected.extend((self.config or {}).get(key) or []) + collected.extend(self.metadata(key) or []) output = [] for url_or_dict in collected: if isinstance(url_or_dict, dict): @@ -2302,11 +1312,6 @@ class Datasette: output.append(script) return output - def _config(self): - return redact_keys( - self.config, ("secret", "key", "password", "token", "hash", "dsn") - ) - def _routes(self): routes = [] @@ -2318,8 +1323,6 @@ class Datasette: routes.append((regex, view)) add_route(IndexView.as_view(self), r"/(\.(?Pjsono?))?$") - add_route(IndexView.as_view(self), r"/-/(\.(?Pjsono?))?$") - add_route(permanent_redirect("/-/"), r"/-$") # TODO: /favicon.ico and /-/static/ deserve far-future cache expires add_route(favicon, "/favicon.ico") @@ -2349,6 +1352,10 @@ class Datasette: ), r"/:memory:(?P.*)$", ) + add_route( + JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()), + r"/-/metadata(\.(?Pjson))?$", + ) add_route( JsonDataView.as_view(self, "versions.json", self._versions), r"/-/versions(\.(?Pjson))?$", @@ -2364,8 +1371,12 @@ class Datasette: r"/-/settings(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view(self, "config.json", lambda: self._config()), - r"/-/config(\.(?Pjson))?$", + permanent_redirect("/-/settings.json"), + r"/-/config.json", + ) + add_route( + permanent_redirect("/-/settings"), + r"/-/config", ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), @@ -2381,16 +1392,6 @@ class Datasette: ), r"/-/actor(\.(?Pjson))?$", ) - add_route( - JsonDataView.as_view( - self, - "actions.json", - self._actions, - template="debug_actions.html", - permission="permissions-debug", - ), - r"/-/actions(\.(?Pjson))?$", - ) add_route( AuthTokenView.as_view(self), r"/-/auth-token$", @@ -2403,18 +1404,6 @@ class Datasette: ApiExplorerView.as_view(self), r"/-/api$", ) - add_route( - JumpView.as_view(self), - r"/-/jump(\.(?Pjson))?$", - ) - add_route( - GlobalQueryListView.as_view(self), - r"/-/queries(\.(?Pjson))?$", - ) - add_route( - InstanceSchemaView.as_view(self), - r"/-/schema(\.(?Pjson|md))?$", - ) add_route( LogoutView.as_view(self), r"/-/logout$", @@ -2423,18 +1412,6 @@ class Datasette: PermissionsDebugView.as_view(self), r"/-/permissions$", ) - add_route( - AllowedResourcesView.as_view(self), - r"/-/allowed(\.(?Pjson))?$", - ) - add_route( - PermissionRulesView.as_view(self), - r"/-/rules(\.(?Pjson))?$", - ) - add_route( - PermissionCheckView.as_view(self), - r"/-/check(\.(?Pjson))?$", - ) add_route( MessagesDebugView.as_view(self), r"/-/messages$", @@ -2456,50 +1433,6 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") - add_route( - QueryListView.as_view(self), - r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", - ) - add_route( - QueryCreateAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/analyze$", - ) - add_route( - QueryStoreView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/store$", - ) - add_route( - ExecuteWriteAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write/analyze$", - ) - add_route( - ExecuteWriteView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write$", - ) - add_route( - DatabaseSchemaView.as_view(self), - r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", - ) - add_route( - QueryParametersView.as_view(self), - r"/(?P[^\/\.]+)/-/query/parameters$", - ) - add_route( - wrap_view(QueryView, self), - r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", - ) - add_route( - QueryDefinitionView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", - ) - add_route( - QueryUpdateView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", - ) - add_route( - QueryDeleteView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/delete$", - ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", @@ -2516,18 +1449,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/upsert$", ) - add_route( - TableSetColumnTypeView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", - ) add_route( TableDropView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", ) - add_route( - TableSchemaView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/schema(\.(?Pjson|md))?$", - ) add_route( RowDeleteView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/delete$", @@ -2547,7 +1472,9 @@ class Datasette: try: return self.get_database(route=database_route) except KeyError: - raise DatabaseNotFound(database_route) + raise DatabaseNotFound( + "Database not found: {}".format(database_route), database_route + ) async def resolve_table(self, request): db = await self.resolve_database(request) @@ -2558,7 +1485,9 @@ class Datasette: if not table_exists: is_view = await db.view_exists(table_name) if not (table_exists or is_view): - raise TableNotFound(db.name, table_name) + raise TableNotFound( + "Table not found: {}".format(table_name), db.name, table_name + ) return ResolvedTable(db, table_name, is_view) async def resolve_row(self, request): @@ -2568,12 +1497,15 @@ class Datasette: results = await db.execute(sql, params, truncate=True) row = results.first() if row is None: - raise RowNotFound(db.name, table_name, pk_values) + raise RowNotFound( + "Row not found: {}".format(pk_values), db.name, table_name, pk_values + ) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() + self._register_custom_units() async def setup_db(): # First time server starts up, calculate table counts for immutable databases @@ -2581,13 +1513,17 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) - async def _close_on_shutdown(): - self.close() - - asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) + asgi = asgi_csrf.asgi_csrf( + DatasetteRouter(self, routes), + signing_secret=self._secret, + cookie_name="ds_csrftoken", + skip_if_scope=lambda scope: any( + pm.hook.skip_csrf(datasette=self, scope=scope) + ), + ) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiLifespan(asgi) asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) @@ -2598,6 +1534,16 @@ class DatasetteRouter: def __init__(self, datasette, routes): self.ds = datasette self.routes = routes or [] + # Build a list of pages/blah/{name}.html matching expressions + pattern_templates = [ + filepath + for filepath in self.ds.jinja_env.list_templates() + if "{" in filepath and filepath.startswith("pages/") + ] + self.page_routes = [ + (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) + for filepath in pattern_templates + ] async def __call__(self, scope, receive, send): # Because we care about "foo/bar" v.s. "foo%2Fbar" we decode raw_path ourselves @@ -2634,13 +1580,10 @@ class DatasetteRouter: # Handle authentication default_actor = scope.get("actor") or None actor = None - results = pm.hook.actor_from_request(datasette=self.ds, request=request) - for result in results: - result = await await_me_maybe(result) - if result and actor is None: - actor = result - # Don't break — we must await all coroutines to avoid - # "coroutine was never awaited" warnings + for actor in pm.hook.actor_from_request(datasette=self.ds, request=request): + actor = await await_me_maybe(actor) + if actor: + break scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) @@ -2700,24 +1643,13 @@ class DatasetteRouter: route_path = request.scope.get("route_path", request.scope["path"]) # Jinja requires template names to use "/" even on Windows template_name = "pages" + route_path + ".html" - # Build a list of pages/blah/{name}.html matching expressions - environment = self.ds.get_jinja_environment(request) - pattern_templates = [ - filepath - for filepath in environment.list_templates() - if "{" in filepath and filepath.startswith("pages/") - ] - page_routes = [ - (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) - for filepath in pattern_templates - ] try: - template = environment.select_template([template_name]) + template = self.ds.jinja_env.select_template([template_name]) except TemplateNotFound: template = None if template is None: # Try for a pages/blah/{name}.html template match - for regex, wildcard_template in page_routes: + for regex, wildcard_template in self.page_routes: match = regex.match(route_path) if match is not None: context.update(match.groupdict()) @@ -2905,18 +1837,9 @@ class NotFoundExplicit(NotFound): class DatasetteClient: - """Internal HTTP client for making requests to a Datasette instance. - - Used for testing and for internal operations that need to make HTTP requests - to the Datasette app without going through an actual HTTP server. - """ - def __init__(self, ds): self.ds = ds - - @property - def app(self): - return self.ds.app() + self.app = ds.app() def actor_cookie(self, actor): # Utility method, mainly for tests @@ -2929,102 +1852,37 @@ class DatasetteClient: path = f"http://localhost{path}" return path - def _apply_actor(self, kwargs): - """If ``actor=`` was supplied, convert it into a signed ds_actor cookie.""" - actor = kwargs.pop("actor", None) - if actor is None: - return - cookies = dict(kwargs.get("cookies") or {}) - if "ds_actor" in cookies: - raise TypeError("Cannot pass both actor= and a ds_actor cookie") - cookies["ds_actor"] = self.actor_cookie(actor) - kwargs["cookies"] = cookies + async def get(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.get(self._fix(path), **kwargs) - async def _request(self, method, path, skip_permission_checks=False, **kwargs): - from datasette.permissions import SkipPermissions + async def options(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.options(self._fix(path), **kwargs) - self._apply_actor(kwargs) - with _DatasetteClientContext(): - if skip_permission_checks: - with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await getattr(client, method)(self._fix(path), **kwargs) - else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await getattr(client, method)(self._fix(path), **kwargs) + async def head(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.head(self._fix(path), **kwargs) - async def get(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "get", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def post(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.post(self._fix(path), **kwargs) - async def options(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "options", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def put(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.put(self._fix(path), **kwargs) - async def head(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "head", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def patch(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.patch(self._fix(path), **kwargs) - async def post(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "post", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def put(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "put", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def patch(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "patch", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def delete(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "delete", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def request(self, method, path, skip_permission_checks=False, **kwargs): - """Make an HTTP request with the specified method. - - Args: - method: HTTP method (e.g., "GET", "POST", "PUT") - path: The path to request - skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx - - Returns: - httpx.Response: The response from the request - """ - from datasette.permissions import SkipPermissions + async def delete(self, path, **kwargs): + async with httpx.AsyncClient(app=self.app) as client: + return await client.delete(self._fix(path), **kwargs) + async def request(self, method, path, **kwargs): avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) - self._apply_actor(kwargs) - with _DatasetteClientContext(): - if skip_permission_checks: - with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.request( - method, self._fix(path, avoid_path_rewrites), **kwargs - ) - else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.request( - method, self._fix(path, avoid_path_rewrites), **kwargs - ) + async with httpx.AsyncClient(app=self.app) as client: + return await client.request( + method, self._fix(path, avoid_path_rewrites), **kwargs + ) diff --git a/datasette/cli.py b/datasette/cli.py index 93aa22ef..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -15,6 +15,7 @@ import sys import textwrap import webbrowser from .app import ( + OBSOLETE_SETTINGS, Datasette, DEFAULT_SETTINGS, SETTINGS, @@ -25,7 +26,6 @@ from .utils import ( LoadExtension, StartupError, check_connection, - deep_dict_update, find_spatialite, parse_metadata, ConnectionProblem, @@ -42,18 +42,6 @@ from .utils.sqlite import sqlite3 from .utils.testing import TestClient from .version import __version__ - -def run_sync(coro_func): - """Run an async callable to completion on a fresh event loop.""" - loop = asyncio.new_event_loop() - try: - asyncio.set_event_loop(loop) - return loop.run_until_complete(coro_func()) - finally: - asyncio.set_event_loop(None) - loop.close() - - # Use Rich for tracebacks if it is installed try: from rich.traceback import install @@ -97,7 +85,7 @@ def sqlite_extensions(fn): "sqlite_extensions", "--load-extension", type=LoadExtension(), - envvar="DATASETTE_LOAD_EXTENSION", + envvar="SQLITE_EXTENSIONS", multiple=True, help="Path to a SQLite extension to load, and optional entrypoint", )(fn) @@ -109,11 +97,15 @@ def sqlite_extensions(fn): return fn(*args, **kwargs) except AttributeError as e: if "enable_load_extension" in str(e): - raise click.ClickException(textwrap.dedent(""" + raise click.ClickException( + textwrap.dedent( + """ Your Python installation does not have the ability to load SQLite extensions. More information: https://datasette.io/help/extensions - """).strip()) + """ + ).strip() + ) raise return wrapped @@ -142,7 +134,9 @@ def inspect(files, inspect_file, sqlite_extensions): This can then be passed to "datasette --inspect-file" to speed up count operations against immutable database files. """ - inspect_data = run_sync(lambda: inspect_(files, sqlite_extensions)) + app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) + loop = asyncio.get_event_loop() + inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions)) if inspect_file == "-": sys.stdout.write(json.dumps(inspect_data, indent=2)) else: @@ -434,20 +428,10 @@ def uninstall(packages, yes): help="Output URL that sets a cookie authenticating the root user", is_flag=True, ) -@click.option( - "--default-deny", - help="Deny all permissions by default", - is_flag=True, -) @click.option( "--get", help="Run an HTTP GET request against this path, print results and exit", ) -@click.option( - "--headers", - is_flag=True, - help="Include HTTP headers in --get output", -) @click.option( "--token", help="API token to send with --get requests", @@ -484,12 +468,10 @@ def uninstall(packages, yes): @click.option( "--ssl-keyfile", help="SSL key file", - envvar="DATASETTE_SSL_KEYFILE", ) @click.option( "--ssl-certfile", help="SSL certificate file", - envvar="DATASETTE_SSL_CERTFILE", ) @click.option( "--internal", @@ -515,9 +497,7 @@ def serve( settings, secret, root, - default_deny, get, - headers, token, actor, version_note, @@ -547,7 +527,7 @@ def serve( if reload: import hupper - reloader = hupper.start_reloader("datasette.cli.cli") + reloader = hupper.start_reloader("datasette.cli.serve") if immutable: reloader.watch_files(immutable) if config: @@ -573,9 +553,7 @@ def serve( # Merge in settings from -s/--setting if settings: settings_updates = pairs_to_nested_config(settings) - # Merge recursively, to avoid over-writing nested values - # https://github.com/simonw/datasette/issues/2389 - deep_dict_update(config_data, settings_updates) + config_data.update(settings_updates) kwargs = dict( immutables=immutable, @@ -596,28 +574,18 @@ def serve( crossdb=crossdb, nolock=nolock, internal=internal, - default_deny=default_deny, ) - # Separate directories from files - directories = [f for f in files if os.path.isdir(f)] - file_paths = [f for f in files if not os.path.isdir(f)] - - # Handle config_dir - only one directory allowed - if len(directories) > 1: - raise click.ClickException( - "Cannot pass multiple directories. Pass a single directory as config_dir." - ) - elif len(directories) == 1: - kwargs["config_dir"] = pathlib.Path(directories[0]) + # if files is a single directory, use that as config_dir= + if 1 == len(files) and os.path.isdir(files[0]): + kwargs["config_dir"] = pathlib.Path(files[0]) + files = [] # Verify list of files, create if needed (and --create) - for file in file_paths: + for file in files: if not pathlib.Path(file).exists(): if create: - conn = sqlite3.connect(file) - conn.execute("vacuum") - conn.close() + sqlite3.connect(file).execute("vacuum") else: raise click.ClickException( "Invalid value for '[FILES]...': Path '{}' does not exist.".format( @@ -625,32 +593,8 @@ def serve( ) ) - # Check for duplicate files by resolving all paths to their absolute forms - # Collect all database files that will be loaded (explicit files + config_dir files) - all_db_files = [] - - # Add explicit files - for file in file_paths: - all_db_files.append((file, pathlib.Path(file).resolve())) - - # Add config_dir databases if config_dir is set - if "config_dir" in kwargs: - config_dir = kwargs["config_dir"] - for ext in ("db", "sqlite", "sqlite3"): - for db_file in config_dir.glob(f"*.{ext}"): - all_db_files.append((str(db_file), db_file.resolve())) - - # Check for duplicates - seen = {} - for original_path, resolved_path in all_db_files: - if resolved_path in seen: - raise click.ClickException( - f"Duplicate database file: '{original_path}' and '{seen[resolved_path]}' " - f"both refer to {resolved_path}" - ) - seen[resolved_path] = original_path - - files = file_paths + # De-duplicate files so 'datasette db.db db.db' only attaches one /db + files = list(dict.fromkeys(files)) try: ds = Datasette(files, **kwargs) @@ -663,43 +607,25 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run async soundness checks before startup hooks, since invoke_startup - # now populates internal tables which requires querying each database - run_sync(lambda: check_databases(ds)) - # Run the "startup" plugin hooks - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) + asyncio.get_event_loop().run_until_complete(ds.invoke_startup()) - if headers and not get: - raise click.ClickException("--headers can only be used with --get") + # Run async soundness checks - but only if we're not under pytest + asyncio.get_event_loop().run_until_complete(check_databases(ds)) if token and not get: raise click.ClickException("--token can only be used with --get") if get: client = TestClient(ds) - request_headers = {} + headers = {} if token: - request_headers["Authorization"] = "Bearer {}".format(token) + headers["Authorization"] = "Bearer {}".format(token) cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) - response = client.get(get, headers=request_headers, cookies=cookies) - - if headers: - # Output HTTP status code, headers, two newlines, then the response body - click.echo(f"HTTP/1.1 {response.status}") - for key, value in response.headers.items(): - click.echo(f"{key}: {value}") - if response.text: - click.echo() - click.echo(response.text) - else: - click.echo(response.text) - + response = client.get(get, headers=headers, cookies=cookies) + click.echo(response.text) exit_code = 0 if response.status == 200 else 1 sys.exit(exit_code) return @@ -707,7 +633,6 @@ def serve( # Start the server url = None if root: - ds.root_enabled = True url = "http://{}:{}{}?token={}".format( host, port, ds.urls.path("-/auth-token"), ds._root_token ) @@ -715,7 +640,9 @@ def serve( if open_browser: if url is None: # Figure out most convenient URL - to table, database or homepage - path = run_sync(lambda: initial_path_for_datasette(ds)) + path = asyncio.get_event_loop().run_until_complete( + initial_path_for_datasette(ds) + ) url = f"http://{host}:{port}{path}" webbrowser.open(url) uvicorn_kwargs = dict( @@ -817,10 +744,8 @@ def create_token( ds = Datasette(secret=secret, plugins_dir=plugins_dir) # Run ds.invoke_startup() in an event loop - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) + loop = asyncio.get_event_loop() + loop.run_until_complete(ds.invoke_startup()) # Warn about any unknown actions actions = [] @@ -828,30 +753,28 @@ def create_token( actions.extend([p[1] for p in databases]) actions.extend([p[2] for p in resources]) for action in actions: - if not ds.actions.get(action): + if not ds.permissions.get(action): click.secho( f" Unknown permission: {action} ", fg="red", err=True, ) - from datasette.tokens import TokenRestrictions - - restrictions = TokenRestrictions() - for action in alls: - restrictions.allow_all(action) + restrict_database = {} for database, action in databases: - restrictions.allow_database(database, action) + restrict_database.setdefault(database, []).append(action) + restrict_resource = {} for database, resource, action in resources: - restrictions.allow_resource(database, resource, action) - - token = run_sync( - lambda: ds.create_token( - id, - expires_after=expires_after, - restrictions=restrictions, - handler="signed", + restrict_resource.setdefault(database, {}).setdefault(resource, []).append( + action ) + + token = ds.create_token( + id, + expires_after=expires_after, + restrict_all=alls, + restrict_database=restrict_database, + restrict_resource=restrict_resource, ) click.echo(token) if debug: diff --git a/datasette/column_types.py b/datasette/column_types.py deleted file mode 100644 index 7320e1d6..00000000 --- a/datasette/column_types.py +++ /dev/null @@ -1,83 +0,0 @@ -from enum import Enum - - -class SQLiteType(Enum): - TEXT = "TEXT" - INTEGER = "INTEGER" - REAL = "REAL" - BLOB = "BLOB" - NULL = "NULL" - - @classmethod - def from_declared_type(cls, declared_type: str | None) -> "SQLiteType | None": - if declared_type is None: - return cls.NULL - - normalized = declared_type.strip().upper() - if not normalized: - return cls.NULL - - if normalized == cls.NULL.value: - return cls.NULL - if "INT" in normalized: - return cls.INTEGER - if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")): - return cls.TEXT - if "BLOB" in normalized: - return cls.BLOB - if any( - token in normalized - for token in ("REAL", "FLOA", "DOUB") # codespell:ignore doub - ): - return cls.REAL - - return None - - -class ColumnType: - """ - Base class for column types. - - Subclasses must define ``name`` and ``description`` as class attributes: - - - ``name``: Unique identifier string. Lowercase, no spaces. - Examples: "markdown", "file", "email", "url", "point", "image". - - ``description``: Human-readable label for admin UI dropdowns. - Examples: "Markdown text", "File reference", "Email address". - - ``sqlite_types``: Optional tuple of SQLiteType values restricting - which SQLite column types this ColumnType can be assigned to. - - Instantiate with an optional ``config`` dict to bind per-column - configuration:: - - ct = MyColumnType(config={"key": "value"}) - ct.config # {"key": "value"} - """ - - name: str - description: str - sqlite_types: tuple[SQLiteType, ...] | None = None - - def __init__(self, config=None): - self.config = config - - async def render_cell(self, value, column, table, database, datasette, request): - """ - Return an HTML string to render this cell value, or None to - fall through to the default render_cell plugin hook chain. - """ - return None - - async def validate(self, value, datasette): - """ - Validate a value before it is written. Return None if valid, - or a string error message if invalid. - """ - return None - - async def transform_value(self, value, datasette): - """ - Transform a value before it appears in JSON API output. - Return the transformed value. Default: return unchanged. - """ - return value diff --git a/datasette/csrf.py b/datasette/csrf.py deleted file mode 100644 index df239aee..00000000 --- a/datasette/csrf.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Header-based CSRF (Cross-Origin) protection. - -Datasette uses the Sec-Fetch-Site + Origin header approach described in -Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented -in Go 1.25's http.CrossOriginProtection. This replaces the previous -token-based asgi-csrf mechanism. -""" - -from __future__ import annotations - -import secrets -import urllib.parse - -from .utils.asgi import asgi_send - -SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) - -DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443} - - -def _normalize_headers(raw_headers): - """Lowercase header names; for duplicates, last value wins.""" - result = {} - for name, value in raw_headers: - if isinstance(name, str): - name = name.encode("latin-1") - if isinstance(value, str): - value = value.encode("latin-1") - result[name.lower()] = value - return result - - -def _origin_tuple(value): - """ - Parse an origin-like string into ``(scheme, host, port)`` with default - ports filled in. Raises ``ValueError`` for malformed input. - """ - parsed = urllib.parse.urlsplit(value) - scheme = (parsed.scheme or "").lower() - host = (parsed.hostname or "").lower() - if not scheme or not host: - raise ValueError("missing scheme or host in {!r}".format(value)) - port = parsed.port # may raise ValueError on bad ports - if port is None: - port = DEFAULT_PORTS.get(scheme) - if port is None: - raise ValueError("unknown default port for scheme {!r}".format(scheme)) - return scheme, host, port - - -def _install_legacy_csrftoken(scope): - """ - Populate ``scope["csrftoken"]`` with a callable returning a per-request - random token. Provided for plugin compatibility only - core no longer - uses this value for CSRF enforcement. - """ - - def csrftoken(): - if "_datasette_legacy_csrftoken" not in scope: - scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32) - return scope["_datasette_legacy_csrftoken"] - - scope["csrftoken"] = csrftoken - - -class CrossOriginProtectionMiddleware: - """ - Modern CSRF protection using the Sec-Fetch-Site and Origin headers. - - Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's - http.CrossOriginProtection. See https://words.filippo.io/csrf/ - - Unsafe-method requests are allowed through only if they look same-origin. - Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin - and are passed through unchanged - CSRF is a browser-only attack. - """ - - SAFE_METHODS = SAFE_METHODS - - def __init__(self, app, datasette): - self.app = app - self.datasette = datasette - - async def __call__(self, scope, receive, send): - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - _install_legacy_csrftoken(scope) - - if scope.get("method", "GET") in self.SAFE_METHODS: - await self.app(scope, receive, send) - return - - headers = _normalize_headers(scope.get("headers") or []) - - authorization = headers.get(b"authorization", b"").decode("latin-1") - cookie_header = headers.get(b"cookie") - # Bearer-token requests are not ambient browser credentials, so they - # are not CSRF-vulnerable. Narrowly exempt them from the header check - # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; - # schemes like Basic or Digest can be browser-managed and ambient. - # If the request also carries a Cookie header, ambient cookie auth - # could be in play, so do NOT treat it as exempt. - if authorization and not cookie_header: - parts = authorization.split(None, 1) - if parts and parts[0].lower() == "bearer": - await self.app(scope, receive, send) - return - - origin_bytes = headers.get(b"origin") - sec_fetch_site_bytes = headers.get(b"sec-fetch-site") - host_bytes = headers.get(b"host", b"") - origin = origin_bytes.decode("latin-1") if origin_bytes else None - sec_fetch_site = ( - sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None - ) - host = host_bytes.decode("latin-1") - - # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS) - if sec_fetch_site is not None: - if sec_fetch_site in ("same-origin", "none"): - await self.app(scope, receive, send) - return - await self._forbid( - send, - "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( - sec_fetch_site - ), - ) - return - - # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.) - if origin is None: - await self.app(scope, receive, send) - return - - # Fallback for older browsers: Origin must match the request's own - # scheme + host + port. Compare full origin tuples, not host alone. - request_scheme = self._request_scheme(scope) - try: - origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) - except ValueError: - await self._forbid( - send, - "Malformed Origin {!r} or Host {!r}".format(origin, host), - ) - return - - if origin_tuple == expected_tuple: - await self.app(scope, receive, send) - return - - await self._forbid( - send, - "Origin {!r} does not match Host {!r}".format(origin, host), - ) - - def _request_scheme(self, scope): - if self.datasette is not None: - try: - if self.datasette.setting("force_https_urls"): - return "https" - except Exception: - pass - return scope.get("scheme") or "http" - - async def _forbid(self, send, reason): - await asgi_send( - send, - content=await self.datasette.render_template( - "csrf_error.html", {"reason": reason} - ), - status=403, - content_type="text/html; charset=utf-8", - ) diff --git a/datasette/database.py b/datasette/database.py index e7e9527e..cb01301e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,32 +1,25 @@ import asyncio -import atexit from collections import namedtuple -import inspect -import os from pathlib import Path +import hashlib +import janus import queue -import sqlite_utils import sys -import tempfile import threading import uuid from .tracer import trace from .utils import ( - call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, get_all_foreign_keys, get_outbound_foreign_keys, - md5_not_usedforsecurity, sqlite_timelimit, sqlite3, table_columns, table_column_details, ) -from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_version from .inspect import inspect_hash connections = threading.local() @@ -34,18 +27,7 @@ connections = threading.local() AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) -class DatasetteClosedError(RuntimeError): - """Raised when using a Datasette or Database instance after close().""" - - -_SHUTDOWN = object() - - class Database: - # For table counts stop at this many rows: - count_limit = 10000 - _thread_local_id_counter = 1 - def __init__( self, ds, @@ -54,55 +36,27 @@ class Database: is_memory=False, memory_name=None, mode=None, - is_temp_disk=False, ): self.name = None - self._thread_local_id = f"x{self._thread_local_id_counter}" - Database._thread_local_id_counter += 1 self.route = None self.ds = ds self.path = path self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name - self.is_temp_disk = is_temp_disk if memory_name is not None: self.is_memory = True - if is_temp_disk: - fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_") - os.close(fd) - self.path = temp_path - self.is_mutable = True - self.mode = "rwc" - self._wal_enabled = False - atexit.register(self._cleanup_temp_file) - else: - self._wal_enabled = False self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None self._write_queue = None - self._closed = False - self._pending_execute_futures = set() - self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] - if not is_temp_disk: - self.mode = mode - - def _check_not_closed(self): - if self._closed: - raise DatasetteClosedError( - "Database {!r} has been closed".format(self.name) - ) - - def _remove_pending_execute_future(self, future): - with self._pending_execute_futures_lock: - self._pending_execute_futures.discard(future) + self.mode = mode @property def cached_table_counts(self): @@ -120,11 +74,9 @@ class Database: def color(self): if self.hash: return self.hash[:6] - return md5_not_usedforsecurity(self.name)[:6] + return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] def suggest_name(self): - if self.is_temp_disk: - return "_temp_disk" if self.path: return Path(self.path).stem elif self.memory_name: @@ -133,13 +85,12 @@ class Database: return "db" def connect(self, write=False): - extra_kwargs = {} - if write: - extra_kwargs["isolation_level"] = "IMMEDIATE" if self.memory_name: uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) conn = sqlite3.connect( - uri, uri=True, check_same_thread=False, **extra_kwargs + uri, + uri=True, + check_same_thread=False, ) if not write: conn.execute("PRAGMA query_only=1") @@ -160,107 +111,35 @@ class Database: if self.mode is not None: qs = f"?mode={self.mode}" conn = sqlite3.connect( - f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs + f"file:{self.path}{qs}", uri=True, check_same_thread=False ) self._all_file_connections.append(conn) - if self.is_temp_disk and not self._wal_enabled: - conn.execute("PRAGMA journal_mode=WAL") - self._wal_enabled = True return conn def close(self): - """Release all resources held by this database. - - Idempotent. After close() further calls to execute()/execute_fn()/ - execute_write()/execute_write_fn() raise DatasetteClosedError. - """ - if self._closed: - return - with self._pending_execute_futures_lock: - if self._closed: - return - self._closed = True - pending_execute_futures = tuple(self._pending_execute_futures) - # Shut down the write thread, if any, via a sentinel. The thread - # drains any writes already queued before the sentinel and then - # closes its own write connection and returns. - write_thread = self._write_thread - if write_thread is not None and self._write_queue is not None: - self._write_queue.put(_SHUTDOWN) - write_thread.join(timeout=10) - if write_thread.is_alive(): - sys.stderr.write( - "Datasette: write thread for {!r} did not exit within 10s\n".format( - self.name - ) - ) - sys.stderr.flush() - for future in pending_execute_futures: - try: - future.result() - except Exception: - pass - # Close anything still tracked in _all_file_connections + # Close all connections - useful to avoid running out of file handles in tests for connection in self._all_file_connections: - try: - connection.close() - except Exception: - pass - self._all_file_connections = [] - # Drop per-thread cached read connections we can reach - try: - delattr(connections, self._thread_local_id) - except AttributeError: - pass - # Close non-threaded-mode cached connections if still open - if self._read_connection is not None: - try: - self._read_connection.close() - except Exception: - pass - self._read_connection = None - if self._write_connection is not None: - try: - self._write_connection.close() - except Exception: - pass - self._write_connection = None - if self.is_temp_disk: - self._cleanup_temp_file() - - def _cleanup_temp_file(self): - if self.is_temp_disk and self.path: - for suffix in ("", "-wal", "-shm"): - try: - os.unlink(self.path + suffix) - except OSError: - pass - - async def execute_write(self, sql, params=None, block=True, request=None): - self._check_not_closed() + connection.close() + async def execute_write(self, sql, params=None, block=True): def _inner(conn): - return conn.execute(sql, params or []) + with conn: + return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn(_inner, block=block, request=request) + results = await self.execute_write_fn(_inner, block=block) return results - async def execute_write_script(self, sql, block=True, request=None): - self._check_not_closed() - + async def execute_write_script(self, sql, block=True): def _inner(conn): - return conn.executescript(sql) + with conn: + return conn.executescript(sql) with trace("sql", database=self.name, sql=sql.strip(), executescript=True): - results = await self.execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + results = await self.execute_write_fn(_inner, block=block) return results - async def execute_write_many(self, sql, params_seq, block=True, request=None): - self._check_not_closed() - + async def execute_write_many(self, sql, params_seq, block=True): def _inner(conn): count = 0 @@ -270,145 +149,43 @@ class Database: count += 1 yield param - return conn.executemany(sql, count_params(params_seq)), count + with conn: + return conn.executemany(sql, count_params(params_seq)), count with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - results, count = await self.execute_write_fn( - _inner, block=block, request=request - ) + results, count = await self.execute_write_fn(_inner, block=block) kwargs["count"] = count return results - async def execute_isolated_fn(self, fn): - self._check_not_closed() - # Open a new connection just for the duration of this function - # blocking the write queue to avoid any writes occurring during it - if self.ds.executor is None: - # non-threaded mode - isolated_connection = self.connect(write=True) - try: - result = fn(isolated_connection) - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - return result - else: - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) - - async def analyze_sql(self, sql, params=None) -> SQLAnalysis: - self._check_not_closed() - - return await self.execute_isolated_fn( - lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) - ) - - async def execute_write_fn(self, fn, block=True, transaction=True, request=None): - self._check_not_closed() - pending_events = [] - - def track_event(event): - pending_events.append(event) - - fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event) + async def execute_write_fn(self, fn, block=True): if self.ds.executor is None: # non-threaded mode if self._write_connection is None: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) - if transaction: - with self._write_connection: - result = fn(self._write_connection) - else: - result = fn(self._write_connection) - else: - result = await self._send_to_write_thread( - fn, block=block, transaction=transaction - ) - if block: - for event in pending_events: - await self.ds.track_event(event) - else: - # For non-blocking writes, spawn a background task to - # dispatch events after the write thread completes - task_id, reply_future = result + return fn(self._write_connection) - async def _dispatch_events_after_write(): - try: - await reply_future - except Exception: - # if the write failed, don't emit success events - return - for event in pending_events: - await self.ds.track_event(event) - - asyncio.ensure_future(_dispatch_events_after_write()) - result = task_id - return result - - def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): - from .plugins import pm - - # Wrap fn so it receives track_event if its signature supports it. - # Historically fn was called positionally, so any single-parameter - # name (conn, connection, db, ...) worked. Preserve that by only - # switching to keyword dependency injection when the callback - # explicitly opts in by declaring a `track_event` parameter. - original_fn = fn - - if "track_event" in inspect.signature(original_fn).parameters: - - def fn_with_track_event(conn): - return call_with_supported_arguments( - original_fn, conn=conn, track_event=track_event - ) - - fn = fn_with_track_event - - wrappers = pm.hook.write_wrapper( - datasette=self.ds, - database=self.name, - request=request, - transaction=transaction, - ) - wrappers = [w for w in wrappers if w is not None] - if not wrappers: - return fn - # Build the wrapped fn by nesting context manager generators. - # The first wrapper returned by pluggy is outermost. - for wrapper_factory in reversed(wrappers): - fn = _apply_write_wrapper(fn, wrapper_factory, track_event) - return fn - - async def _send_to_write_thread( - self, fn, block=True, isolated_connection=False, transaction=True - ): + # threaded mode + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) - self._write_thread.name = "_execute_writes for database {}".format( - self.name - ) self._write_thread.start() - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - loop = asyncio.get_running_loop() - reply_future = loop.create_future() - self._write_queue.put( - WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) - ) + reply_queue = janus.Queue() + self._write_queue.put(WriteTask(fn, task_id, reply_queue)) if block: - return await reply_future + result = await reply_queue.async_q.get() + if isinstance(result, Exception): + raise result + else: + return result else: - return task_id, reply_future + return task_id def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -422,47 +199,18 @@ class Database: conn_exception = e while True: task = self._write_queue.get() - if task is _SHUTDOWN: - if conn is not None: - try: - conn.close() - except Exception: - pass - return - exception = None - result = None if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass + result = conn_exception else: try: - if task.transaction: - with conn: - result = task.fn(conn) - else: - result = task.fn(conn) + result = task.fn(conn) except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) + result = e + task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): - self._check_not_closed() if self.ds.executor is None: # non-threaded mode if self._read_connection is None: @@ -472,19 +220,16 @@ class Database: # threaded mode def in_thread(): - conn = getattr(connections, self._thread_local_id, None) + conn = getattr(connections, self.name, None) if not conn: conn = self.connect() self.ds._prepare_connection(conn, self.name) - setattr(connections, self._thread_local_id, conn) + setattr(connections, self.name, conn) return fn(conn) - with self._pending_execute_futures_lock: - self._check_not_closed() - future = self.ds.executor.submit(in_thread) - self._pending_execute_futures.add(future) - future.add_done_callback(self._remove_pending_execute_future) - return await asyncio.wrap_future(future) + return await asyncio.get_event_loop().run_in_executor( + self.ds.executor, in_thread + ) async def execute( self, @@ -496,7 +241,6 @@ class Database: log_sql_errors=True, ): """Executes sql against db_name in a thread""" - self._check_not_closed() page_size = page_size or self.ds.page_size def sql_operation_in_thread(conn): @@ -544,7 +288,7 @@ class Database: def hash(self): if self.cached_hash is not None: return self.cached_hash - elif self.is_mutable or self.is_memory or self.is_temp_disk: + elif self.is_mutable or self.is_memory: return None elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): self.cached_hash = self.ds.inspect_data[self.name]["hash"] @@ -578,7 +322,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", + f"select count(*) from [{table}]", custom_time_limit=limit, ) ).rows[0][0] @@ -603,12 +347,7 @@ class Database: # But SQLite prior to 3.16.0 doesn't support pragma functions results = await self.execute("PRAGMA database_list;") # {'seq': 0, 'name': 'main', 'file': ''} - return [ - AttachedDatabase(*row) - for row in results.rows - # Filter out the SQLite internal "temp" database, refs #2557 - if row["seq"] > 0 and row["name"] != "temp" - ] + return [AttachedDatabase(*row) for row in results.rows if row["seq"] > 0] async def table_exists(self, table): results = await self.execute( @@ -624,7 +363,7 @@ class Database: async def table_names(self): results = await self.execute( - "select name from sqlite_master where type='table' order by name" + "select name from sqlite_master where type='table'" ) return [r[0] for r in results.rows] @@ -641,38 +380,12 @@ class Database: return await self.execute_fn(lambda conn: detect_fts(conn, table)) async def label_column_for_table(self, table): - explicit_label_column = (await self.ds.table_config(self.name, table)).get( + explicit_label_column = self.ds.table_metadata(self.name, table).get( "label_column" ) if explicit_label_column: return explicit_label_column - - def column_details(conn): - # Returns {column_name: (type, is_unique)} - db = sqlite_utils.Database(conn) - columns = db[table].columns_dict - indexes = db[table].indexes - details = {} - for name in columns: - is_unique = any( - index - for index in indexes - if index.columns == [name] and index.unique - ) - details[name] = (columns[name], is_unique) - return details - - column_details = await self.execute_fn(column_details) - # Is there just one unique column that's text? - unique_text_columns = [ - name - for name, (type_, is_unique) in column_details.items() - if is_unique and type_ is str - ] - if len(unique_text_columns) == 1: - return unique_text_columns[0] - - column_names = list(column_details.keys()) + column_names = await self.execute_fn(lambda conn: table_columns(conn, table)) # Is there a name or title column? name_or_title = [c for c in column_names if c.lower() in ("name", "title")] if name_or_title: @@ -682,7 +395,6 @@ class Database: column_names and len(column_names) == 2 and ("id" in column_names or "pk" in column_names) - and not set(column_names) == {"id", "pk"} ): return [c for c in column_names if c not in ("id", "pk")][0] # Couldn't find a label: @@ -694,92 +406,21 @@ class Database: ) async def hidden_table_names(self): - hidden_tables = [] - # Add any tables marked as hidden in config - db_config = self.ds.config.get("databases", {}).get(self.name, {}) - if "tables" in db_config: - hidden_tables += [ - t for t in db_config["tables"] if db_config["tables"][t].get("hidden") - ] - - if sqlite_version()[1] >= 37: - hidden_tables += [x[0] for x in await self.execute(""" - with shadow_tables as ( - select name - from pragma_table_list - where [type] = 'shadow' - order by name - ), - core_tables as ( - select name - from sqlite_master - WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - combined as ( - select name from shadow_tables - union all - select name from core_tables - ) - select name from combined order by 1 - """)] - else: - hidden_tables += [x[0] for x in await self.execute(""" - WITH base AS ( - SELECT name - FROM sqlite_master - WHERE name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - fts_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config')) - ), - fts5_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%' - ), - fts5_shadow_tables AS ( - SELECT - printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name - FROM fts5_names - JOIN fts_suffixes - ), - fts3_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize')) - ), - fts3_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%' - OR sql LIKE '%VIRTUAL TABLE%USING FTS4%' - ), - fts3_shadow_tables AS ( - SELECT - printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name - FROM fts3_names - JOIN fts3_suffixes - ), - final AS ( - SELECT name FROM base - UNION ALL - SELECT name FROM fts5_shadow_tables - UNION ALL - SELECT name FROM fts3_shadow_tables - ) - SELECT name FROM final ORDER BY 1 - """)] - # Also hide any FTS tables that have a content= argument - hidden_tables += [x[0] for x in await self.execute(""" - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%' - AND sql LIKE '%USING FTS%' - AND sql LIKE '%content=%' - """)] - + # Mark tables 'hidden' if they relate to FTS virtual tables + hidden_tables = [ + r[0] + for r in ( + await self.execute( + """ + select name from sqlite_master + where rootpage = 0 + and ( + sql like '%VIRTUAL TABLE%USING FTS%' + ) or name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') + """ + ) + ).rows + ] has_spatialite = await self.execute_fn(detect_spatialite) if has_spatialite: # Also hide Spatialite internal tables @@ -797,12 +438,32 @@ class Database: "KNN", "KNN2", ] + [ - r[0] for r in (await self.execute(""" + r[0] + for r in ( + await self.execute( + """ select name from sqlite_master where name like "idx_%" and type = "table" - """)).rows + """ + ) + ).rows ] + # Add any from metadata.json + db_metadata = self.ds.metadata(database=self.name) + if "tables" in db_metadata: + hidden_tables += [ + t + for t in db_metadata["tables"] + if db_metadata["tables"][t].get("hidden") + ] + # Also mark as hidden any tables which start with the name of a hidden table + # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden + for table_name in await self.table_names(): + for hidden_table in hidden_tables[:]: + if table_name.startswith(hidden_table): + hidden_tables.append(table_name) + continue return hidden_tables @@ -843,8 +504,6 @@ class Database: tags.append("mutable") if self.is_memory: tags.append("memory") - if self.is_temp_disk: - tags.append("temp_disk") if self.hash: tags.append(f"hash={self.hash}") if self.size is not None: @@ -855,88 +514,13 @@ class Database: return f"" -def _apply_write_wrapper(fn, wrapper_factory, track_event): - """Apply a single write_wrapper context manager around fn. - - ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally - ``track_event``, and returns a generator that yields exactly once. - Code before the yield runs before ``fn(conn)``, code after the yield - runs after. The result of ``fn(conn)`` is sent into the generator - via ``.send()``, and any exception raised by ``fn(conn)`` is thrown - via ``.throw()``. - """ - - def wrapped(conn): - gen = call_with_supported_arguments( - wrapper_factory, conn=conn, track_event=track_event - ) - # Advance to the yield point (run "before" code) - try: - next(gen) - except StopIteration: - # Generator didn't yield — just run fn unchanged - return fn(conn) - - # Execute the actual write - try: - result = fn(conn) - except Exception: - # Throw exception into generator so it can handle it - try: - gen.throw(*sys.exc_info()) - except StopIteration: - pass - # Re-raise the original exception - raise - else: - # Send the result back through the yield - try: - gen.send(result) - except StopIteration: - pass - return result - - return wrapped - - class WriteTask: - __slots__ = ( - "fn", - "task_id", - "loop", - "reply_future", - "isolated_connection", - "transaction", - ) + __slots__ = ("fn", "task_id", "reply_queue") - def __init__( - self, fn, task_id, loop, reply_future, isolated_connection, transaction - ): + def __init__(self, fn, task_id, reply_queue): self.fn = fn self.task_id = task_id - self.loop = loop - self.reply_future = reply_future - self.isolated_connection = isolated_connection - self.transaction = transaction - - -def _deliver_write_result(task, result, exception): - # Called from the write thread. Delivers the result back to the - # awaiting coroutine on its event loop via call_soon_threadsafe. - def _set(): - if task.reply_future.done(): - # Awaiter was cancelled; nothing to do. - return - if exception is not None: - task.reply_future.set_exception(exception) - else: - task.reply_future.set_result(result) - - try: - task.loop.call_soon_threadsafe(_set) - except RuntimeError: - # Event loop has been closed; the awaiter is gone. - pass + self.reply_queue = reply_queue class QueryInterrupted(Exception): @@ -945,9 +529,6 @@ class QueryInterrupted(Exception): self.sql = sql self.params = params - def __str__(self): - return "QueryInterrupted: {}".format(self.e) - class MultipleValues(Exception): pass @@ -975,9 +556,6 @@ class Results: else: raise MultipleValues - def dicts(self): - return [dict(row) for row in self.rows] - def __iter__(self): return iter(self.rows) diff --git a/datasette/default_actions.py b/datasette/default_actions.py deleted file mode 100644 index 2f78570b..00000000 --- a/datasette/default_actions.py +++ /dev/null @@ -1,133 +0,0 @@ -from datasette import hookimpl -from datasette.permissions import Action -from datasette.resources import ( - DatabaseResource, - TableResource, - QueryResource, -) - - -@hookimpl -def register_actions(): - """Register the core Datasette actions.""" - return ( - # Global actions (no resource_class) - Action( - name="view-instance", - abbr="vi", - description="View Datasette instance", - ), - Action( - name="permissions-debug", - abbr="pd", - description="Access permission debug tool", - ), - Action( - name="debug-menu", - abbr="dm", - description="View debug menu items", - ), - # Database-level actions (parent-level) - Action( - name="view-database", - abbr="vd", - description="View database", - resource_class=DatabaseResource, - ), - Action( - name="view-database-download", - abbr="vdd", - description="Download database file", - resource_class=DatabaseResource, - also_requires="view-database", - ), - Action( - name="execute-sql", - abbr="es", - description="Execute read-only SQL queries", - resource_class=DatabaseResource, - also_requires="view-database", - ), - Action( - name="execute-write-sql", - abbr="ews", - description="Execute writable SQL queries", - resource_class=DatabaseResource, - also_requires="view-database", - ), - Action( - name="create-table", - abbr="ct", - description="Create tables", - resource_class=DatabaseResource, - ), - Action( - name="store-query", - abbr="sq", - description="Create stored queries", - resource_class=DatabaseResource, - also_requires="execute-sql", - ), - # Table-level actions (child-level) - Action( - name="view-table", - abbr="vt", - description="View table", - resource_class=TableResource, - ), - Action( - name="insert-row", - abbr="ir", - description="Insert rows", - resource_class=TableResource, - ), - Action( - name="delete-row", - abbr="dr", - description="Delete rows", - resource_class=TableResource, - ), - Action( - name="update-row", - abbr="ur", - description="Update rows", - resource_class=TableResource, - ), - Action( - name="alter-table", - abbr="at", - description="Alter tables", - resource_class=TableResource, - ), - Action( - name="set-column-type", - abbr="sct", - description="Set column type", - resource_class=TableResource, - ), - Action( - name="drop-table", - abbr="dt", - description="Drop tables", - resource_class=TableResource, - ), - # Query-level actions (child-level) - Action( - name="view-query", - abbr="vq", - description="View named query results", - resource_class=QueryResource, - ), - Action( - name="update-query", - abbr="uq", - description="Update stored queries", - resource_class=QueryResource, - ), - Action( - name="delete-query", - abbr="dq", - description="Delete stored queries", - resource_class=QueryResource, - ), - ) diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py deleted file mode 100644 index 24493994..00000000 --- a/datasette/default_column_types.py +++ /dev/null @@ -1,81 +0,0 @@ -import json -import re - -import markupsafe - -from datasette import hookimpl -from datasette.column_types import ColumnType, SQLiteType - - -class UrlColumnType(ColumnType): - name = "url" - description = "URL" - sqlite_types = (SQLiteType.TEXT,) - - async def render_cell(self, value, column, table, database, datasette, request): - if not value or not isinstance(value, str): - return None - escaped = markupsafe.escape(value.strip()) - return markupsafe.Markup(f'{escaped}') - - async def validate(self, value, datasette): - if value is None or value == "": - return None - if not isinstance(value, str): - return "URL must be a string" - if not re.match(r"^https?://\S+$", value.strip()): - return "Invalid URL" - return None - - -class EmailColumnType(ColumnType): - name = "email" - description = "Email address" - sqlite_types = (SQLiteType.TEXT,) - - async def render_cell(self, value, column, table, database, datasette, request): - if not value or not isinstance(value, str): - return None - escaped = markupsafe.escape(value.strip()) - return markupsafe.Markup(f'{escaped}') - - async def validate(self, value, datasette): - if value is None or value == "": - return None - if not isinstance(value, str): - return "Email must be a string" - if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", value.strip()): - return "Invalid email address" - return None - - -class JsonColumnType(ColumnType): - name = "json" - description = "JSON data" - sqlite_types = (SQLiteType.TEXT,) - - async def render_cell(self, value, column, table, database, datasette, request): - if value is None: - return None - try: - parsed = json.loads(value) if isinstance(value, str) else value - formatted = json.dumps(parsed, indent=2) - escaped = markupsafe.escape(formatted) - return markupsafe.Markup(f"
{escaped}
") - except (json.JSONDecodeError, TypeError): - return None - - async def validate(self, value, datasette): - if value is None or value == "": - return None - if isinstance(value, str): - try: - json.loads(value) - except json.JSONDecodeError: - return "Invalid JSON" - return None - - -@hookimpl -def register_column_types(datasette): - return [UrlColumnType, EmailColumnType, JsonColumnType] diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py deleted file mode 100644 index e0cb3cdf..00000000 --- a/datasette/default_database_actions.py +++ /dev/null @@ -1,24 +0,0 @@ -from datasette import hookimpl -from datasette.resources import DatabaseResource - - -@hookimpl -def database_actions(datasette, actor, database, request): - async def inner(): - if not datasette.get_database(database).is_mutable: - return [] - if not await datasette.allowed( - action="execute-write-sql", - resource=DatabaseResource(database), - actor=actor, - ): - return [] - return [ - { - "href": datasette.urls.database(database) + "/-/execute-write", - "label": "Execute write SQL", - "description": "Run writable SQL with table permission checks.", - } - ] - - return inner diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py deleted file mode 100644 index 6127b2a6..00000000 --- a/datasette/default_debug_menu.py +++ /dev/null @@ -1,75 +0,0 @@ -from datasette import hookimpl -from datasette.jump import JumpSQL - -DEBUG_MENU_ITEMS = ( - ( - "/-/databases", - "Databases", - "List of databases known to this Datasette instance.", - ), - ( - "/-/plugins", - "Installed plugins", - "Review loaded plugins, their versions and their registered hooks.", - ), - ( - "/-/versions", - "Version info", - "Check the Python, SQLite and dependency versions used by this server.", - ), - ( - "/-/settings", - "Settings", - "Inspect the active Datasette settings and configuration values.", - ), - ( - "/-/permissions", - "Debug permissions", - "Test permission checks for actors, actions and resources.", - ), - ( - "/-/messages", - "Debug messages", - "Try out temporary flash messages shown to users.", - ), - ( - "/-/allow-debug", - "Debug allow rules", - "Explore how allow blocks match actors against permission rules.", - ), - ( - "/-/threads", - "Debug threads", - "Inspect worker threads and database tasks.", - ), - ( - "/-/actor", - "Debug actor", - "View the actor object for the current signed-in user.", - ), - ( - "/-/patterns", - "Pattern portfolio", - "Browse Datasette UI patterns.", - ), -) - - -@hookimpl -def jump_items_sql(datasette, actor, request): - async def inner(): - if not await datasette.allowed(action="debug-menu", actor=actor): - return [] - - return [ - JumpSQL.menu_item( - label=label, - url=datasette.urls.path(path), - description=description, - search_text=f"debug {label} {description}", - item_type="debug", - ) - for path, label, description in DEBUG_MENU_ITEMS - ] - - return inner diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py deleted file mode 100644 index d215e7ec..00000000 --- a/datasette/default_jump_items.py +++ /dev/null @@ -1,82 +0,0 @@ -from datasette import hookimpl -from datasette.jump import JumpSQL - - -@hookimpl -def jump_items_sql(datasette, actor, request): - async def inner(): - database_sql, database_params = await datasette.allowed_resources_sql( - action="view-database", actor=actor - ) - table_sql, table_params = await datasette.allowed_resources_sql( - action="view-table", actor=actor - ) - query_sql, query_params = await datasette.allowed_resources_sql( - action="view-query", actor=actor - ) - return [ - JumpSQL( - sql=f""" - WITH allowed_databases AS ( - {database_sql} - ) - SELECT - 'database' AS type, - parent AS label, - NULL AS description, - json_object( - 'method', 'database', - 'database', parent - ) AS url, - parent AS search_text, - NULL AS display_name - FROM allowed_databases - """, - params=database_params, - ), - JumpSQL( - sql=f""" - WITH allowed_tables AS ( - {table_sql} - ) - SELECT - CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, - allowed_tables.parent || ': ' || allowed_tables.child AS label, - NULL AS description, - json_object( - 'method', 'table', - 'database', allowed_tables.parent, - 'table', allowed_tables.child - ) AS url, - allowed_tables.parent || ' ' || allowed_tables.child AS search_text, - NULL AS display_name - FROM allowed_tables - LEFT JOIN catalog_views - ON catalog_views.database_name = allowed_tables.parent - AND catalog_views.view_name = allowed_tables.child - """, - params=table_params, - ), - JumpSQL( - sql=f""" - WITH allowed_queries AS ( - {query_sql} - ) - SELECT - 'query' AS type, - allowed_queries.parent || ': ' || allowed_queries.child AS label, - NULL AS description, - json_object( - 'method', 'query', - 'database', allowed_queries.parent, - 'query', allowed_queries.child - ) AS url, - allowed_queries.parent || ' ' || allowed_queries.child AS search_text, - NULL AS display_name - FROM allowed_queries - """, - params=query_params, - ), - ] - - return inner diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index 91c1c5aa..19382207 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -24,12 +24,9 @@ def now(key, request): if key == "epoch": return int(time.time()) elif key == "date_utc": - return datetime.datetime.now(datetime.timezone.utc).date().isoformat() + return datetime.datetime.utcnow().date().isoformat() elif key == "datetime_utc": - return ( - datetime.datetime.now(datetime.timezone.utc).strftime(r"%Y-%m-%dT%H:%M:%S") - + "Z" - ) + return datetime.datetime.utcnow().strftime(r"%Y-%m-%dT%H:%M:%S") + "Z" else: raise KeyError diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py new file mode 100644 index 00000000..56f481ef --- /dev/null +++ b/datasette/default_menu_links.py @@ -0,0 +1,45 @@ +from datasette import hookimpl + + +@hookimpl +def menu_links(datasette, actor): + async def inner(): + if not await datasette.permission_allowed(actor, "debug-menu"): + return [] + + return [ + {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, + { + "href": datasette.urls.path("/-/plugins"), + "label": "Installed plugins", + }, + { + "href": datasette.urls.path("/-/versions"), + "label": "Version info", + }, + { + "href": datasette.urls.path("/-/metadata"), + "label": "Metadata", + }, + { + "href": datasette.urls.path("/-/settings"), + "label": "Settings", + }, + { + "href": datasette.urls.path("/-/permissions"), + "label": "Debug permissions", + }, + { + "href": datasette.urls.path("/-/messages"), + "label": "Debug messages", + }, + { + "href": datasette.urls.path("/-/allow-debug"), + "label": "Debug allow rules", + }, + {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"}, + {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, + {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, + ] + + return inner diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py new file mode 100644 index 00000000..5a99d0d8 --- /dev/null +++ b/datasette/default_permissions.py @@ -0,0 +1,405 @@ +from datasette import hookimpl, Permission +from datasette.utils import actor_matches_allow +import itsdangerous +import time +from typing import Union, Tuple + + +@hookimpl +def register_permissions(): + return ( + # name, abbr, description, takes_database, takes_resource, default + Permission( + name="view-instance", + abbr="vi", + description="View Datasette instance", + takes_database=False, + takes_resource=False, + default=True, + ), + Permission( + name="view-database", + abbr="vd", + description="View database", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="view-database-download", + abbr="vdd", + description="Download database file", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="view-table", + abbr="vt", + description="View table", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="view-query", + abbr="vq", + description="View named query results", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="execute-sql", + abbr="es", + description="Execute read-only SQL queries", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="permissions-debug", + abbr="pd", + description="Access permission debug tool", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="debug-menu", + abbr="dm", + description="View debug menu items", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="insert-row", + abbr="ir", + description="Insert rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="delete-row", + abbr="dr", + description="Delete rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="update-row", + abbr="ur", + description="Update rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="create-table", + abbr="ct", + description="Create tables", + takes_database=True, + takes_resource=False, + default=False, + ), + Permission( + name="drop-table", + abbr="dt", + description="Drop tables", + takes_database=True, + takes_resource=True, + default=False, + ), + ) + + +@hookimpl(tryfirst=True, specname="permission_allowed") +def permission_allowed_default(datasette, actor, action, resource): + async def inner(): + # id=root gets some special permissions: + if action in ( + "permissions-debug", + "debug-menu", + "insert-row", + "create-table", + "drop-table", + "delete-row", + "update-row", + ): + if actor and actor.get("id") == "root": + return True + + # Resolve metadata view permissions + if action in ( + "view-instance", + "view-database", + "view-table", + "view-query", + "execute-sql", + ): + result = await _resolve_metadata_view_permissions( + datasette, actor, action, resource + ) + if result is not None: + return result + + # Check custom permissions: blocks + result = await _resolve_metadata_permissions_blocks( + datasette, actor, action, resource + ) + if result is not None: + return result + + # --setting default_allow_sql + if action == "execute-sql" and not datasette.setting("default_allow_sql"): + return False + + return inner + + +async def _resolve_metadata_permissions_blocks(datasette, actor, action, resource): + # Check custom permissions: blocks + metadata = datasette.metadata() + root_block = (metadata.get("permissions", None) or {}).get(action) + if root_block: + root_result = actor_matches_allow(actor, root_block) + if root_result is not None: + return root_result + # Now try database-specific blocks + if not resource: + return None + if isinstance(resource, str): + database = resource + else: + database = resource[0] + database_block = ( + (metadata.get("databases", {}).get(database, {}).get("permissions", None)) or {} + ).get(action) + if database_block: + database_result = actor_matches_allow(actor, database_block) + if database_result is not None: + return database_result + # Finally try table/query specific blocks + if not isinstance(resource, tuple): + return None + database, table_or_query = resource + table_block = ( + ( + metadata.get("databases", {}) + .get(database, {}) + .get("tables", {}) + .get(table_or_query, {}) + .get("permissions", None) + ) + or {} + ).get(action) + if table_block: + table_result = actor_matches_allow(actor, table_block) + if table_result is not None: + return table_result + # Finally the canned queries + query_block = ( + ( + metadata.get("databases", {}) + .get(database, {}) + .get("queries", {}) + .get(table_or_query, {}) + .get("permissions", None) + ) + or {} + ).get(action) + if query_block: + query_result = actor_matches_allow(actor, query_block) + if query_result is not None: + return query_result + return None + + +async def _resolve_metadata_view_permissions(datasette, actor, action, resource): + if action == "view-instance": + allow = datasette.metadata("allow") + if allow is not None: + return actor_matches_allow(actor, allow) + elif action == "view-database": + database_allow = datasette.metadata("allow", database=resource) + if database_allow is None: + return None + return actor_matches_allow(actor, database_allow) + elif action == "view-table": + database, table = resource + tables = datasette.metadata("tables", database=database) or {} + table_allow = (tables.get(table) or {}).get("allow") + if table_allow is None: + return None + return actor_matches_allow(actor, table_allow) + elif action == "view-query": + # Check if this query has a "allow" block in metadata + database, query_name = resource + query = await datasette.get_canned_query(database, query_name, actor) + assert query is not None + allow = query.get("allow") + if allow is None: + return None + return actor_matches_allow(actor, allow) + elif action == "execute-sql": + # Use allow_sql block from database block, or from top-level + database_allow_sql = datasette.metadata("allow_sql", database=resource) + if database_allow_sql is None: + database_allow_sql = datasette.metadata("allow_sql") + if database_allow_sql is None: + return None + return actor_matches_allow(actor, database_allow_sql) + + +def restrictions_allow_action( + datasette: "Datasette", + restrictions: dict, + action: str, + resource: Union[str, Tuple[str, str]], +): + "Do these restrictions allow the requested action against the requested resource?" + if action == "view-instance": + # Special case for view-instance: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set + all_rules = restrictions.get("a") or [] + for database_rules in (restrictions.get("d") or {}).values(): + all_rules += database_rules + for database_resource_rules in (restrictions.get("r") or {}).values(): + for resource_rules in database_resource_rules.values(): + all_rules += resource_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view): + return True + + if action == "view-database": + # Special case for view-database: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set AND takes_database + all_rules = restrictions.get("a") or [] + database_rules = list((restrictions.get("d") or {}).get(resource) or []) + all_rules += database_rules + resource_rules = ((restrictions.get("r") or {}).get(resource) or {}).values() + for resource_rules in (restrictions.get("r") or {}).values(): + for table_rules in resource_rules.values(): + all_rules += table_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view and p.takes_database): + return True + + # Does this action have an abbreviation? + to_check = {action} + permission = datasette.permissions.get(action) + if permission and permission.abbr: + to_check.add(permission.abbr) + + # If restrictions is defined then we use those to further restrict the actor + # Crucially, we only use this to say NO (return False) - we never + # use it to return YES (True) because that might over-ride other + # restrictions placed on this actor + all_allowed = restrictions.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if to_check.intersection(all_allowed): + return True + # How about for the current database? + if resource: + if isinstance(resource, str): + database_name = resource + else: + database_name = resource[0] + database_allowed = restrictions.get("d", {}).get(database_name) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if to_check.intersection(database_allowed): + return True + # Or the current table? That's any time the resource is (database, table) + if resource is not None and not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + # TODO: What should this do for canned queries? + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True + + # This action is not specifically allowed, so reject it + return False + + +@hookimpl(specname="permission_allowed") +def permission_allowed_actor_restrictions(datasette, actor, action, resource): + if actor is None: + return None + if "_r" not in actor: + # No restrictions, so we have no opinion + return None + _r = actor.get("_r") + if restrictions_allow_action(datasette, _r, action, resource): + # Return None because we do not have an opinion here + return None + else: + # Block this permission check + return False + + +@hookimpl +def actor_from_request(datasette, request): + prefix = "dstok_" + if not datasette.setting("allow_signed_tokens"): + return None + max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") + authorization = request.headers.get("authorization") + if not authorization: + return None + if not authorization.startswith("Bearer "): + return None + token = authorization[len("Bearer ") :] + if not token.startswith(prefix): + return None + token = token[len(prefix) :] + try: + decoded = datasette.unsign(token, namespace="token") + except itsdangerous.BadSignature: + return None + if "t" not in decoded: + # Missing timestamp + return None + created = decoded["t"] + if not isinstance(created, int): + # Invalid timestamp + return None + duration = decoded.get("d") + if duration is not None and not isinstance(duration, int): + # Invalid duration + return None + if (duration is None and max_signed_tokens_ttl) or ( + duration is not None + and max_signed_tokens_ttl + and duration > max_signed_tokens_ttl + ): + duration = max_signed_tokens_ttl + if duration: + if time.time() - created > duration: + # Expired + return None + actor = {"id": decoded["a"], "token": "dstok"} + if "_r" in decoded: + actor["_r"] = decoded["_r"] + if duration: + actor["token_expires"] = created + duration + return actor + + +@hookimpl +def skip_csrf(scope): + # Skip CSRF check for requests with content-type: application/json + if scope["type"] == "http": + headers = scope.get("headers") or {} + if dict(headers).get(b"content-type") == b"application/json": + return True diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py deleted file mode 100644 index 6cd46f04..00000000 --- a/datasette/default_permissions/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Default permission implementations for Datasette. - -This module provides the built-in permission checking logic through implementations -of the permission_resources_sql hook. The hooks are organized by their purpose: - -1. Actor Restrictions - Enforces _r allowlists embedded in actor tokens -2. Root User - Grants full access when --root flag is used -3. Config Rules - Applies permissions from datasette.yaml -4. Default Settings - Enforces default_allow_sql and default view permissions - -IMPORTANT: These hooks return PermissionSQL objects that are combined using SQL -UNION/INTERSECT operations. The order of evaluation is: - - restriction_sql fields are INTERSECTed (all must match) - - Regular sql fields are UNIONed and evaluated with cascading priority -""" - -from __future__ import annotations - -# Re-export all hooks and public utilities -from .restrictions import ( - actor_restrictions_sql as actor_restrictions_sql, - restrictions_allow_action as restrictions_allow_action, - ActorRestrictions as ActorRestrictions, -) -from .root import root_user_permissions_sql as root_user_permissions_sql -from .config import config_permissions_sql as config_permissions_sql -from .defaults import ( - # Avoid "datasette.default_permissions" does not explicitly export attribute - default_allow_sql_check as default_allow_sql_check, - default_action_permissions_sql as default_action_permissions_sql, - default_query_permissions_sql as default_query_permissions_sql, - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, -) diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py deleted file mode 100644 index aab87c1c..00000000 --- a/datasette/default_permissions/config.py +++ /dev/null @@ -1,442 +0,0 @@ -""" -Config-based permission handling for Datasette. - -Applies permission rules from datasette.yaml configuration. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL -from datasette.utils import actor_matches_allow - -from .helpers import PermissionRowCollector, get_action_name_variants - - -class ConfigPermissionProcessor: - """ - Processes permission rules from datasette.yaml configuration. - - Configuration structure: - - permissions: # Root-level permissions block - view-instance: - id: admin - - databases: - mydb: - permissions: # Database-level permissions - view-database: - id: admin - allow: # Database-level allow block (for view-*) - id: viewer - allow_sql: # execute-sql allow block - id: analyst - tables: - users: - permissions: # Table-level permissions - view-table: - id: admin - allow: # Table-level allow block - id: viewer - queries: - my_query: - permissions: # Query-level permissions - view-query: - id: admin - allow: # Query-level allow block - id: viewer - """ - - def __init__( - self, - datasette: "Datasette", - actor: Optional[dict], - action: str, - ): - self.datasette = datasette - self.actor = actor - self.action = action - self.config = datasette.config or {} - self.collector = PermissionRowCollector(prefix="cfg") - - # Pre-compute action variants - self.action_checks = get_action_name_variants(datasette, action) - self.action_obj = datasette.actions.get(action) - - # Parse restrictions if present - self.has_restrictions = actor and "_r" in actor if actor else False - self.restrictions = actor.get("_r", {}) if actor else {} - - # Pre-compute restriction info for efficiency - self.restricted_databases: Set[str] = set() - self.restricted_tables: Set[Tuple[str, str]] = set() - - if self.has_restrictions: - self.restricted_databases = { - db_name - for db_name, db_actions in (self.restrictions.get("d") or {}).items() - if self.action_checks.intersection(db_actions) - } - self.restricted_tables = { - (db_name, table_name) - for db_name, tables in (self.restrictions.get("r") or {}).items() - for table_name, table_actions in tables.items() - if self.action_checks.intersection(table_actions) - } - # Tables implicitly reference their parent databases - self.restricted_databases.update(db for db, _ in self.restricted_tables) - - def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: - """Evaluate an allow block against the current actor.""" - if allow_block is None: - return None - return actor_matches_allow(self.actor, allow_block) - - def is_in_restriction_allowlist( - self, - parent: Optional[str], - child: Optional[str], - ) -> bool: - """Check if resource is allowed by actor restrictions.""" - if not self.has_restrictions: - return True # No restrictions, all resources allowed - - # Check global allowlist - if self.action_checks.intersection(self.restrictions.get("a", [])): - return True - - # Check database-level allowlist - if parent and self.action_checks.intersection( - self.restrictions.get("d", {}).get(parent, []) - ): - return True - - # Check table-level allowlist - if parent: - table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) - if child: - table_actions = table_restrictions.get(child, []) - if self.action_checks.intersection(table_actions): - return True - else: - # Parent query should proceed if any child in this database is allowlisted - for table_actions in table_restrictions.values(): - if self.action_checks.intersection(table_actions): - return True - - # Parent/child both None: include if any restrictions exist for this action - if parent is None and child is None: - if self.action_checks.intersection(self.restrictions.get("a", [])): - return True - if self.restricted_databases: - return True - if self.restricted_tables: - return True - - return False - - def add_permissions_rule( - self, - parent: Optional[str], - child: Optional[str], - permissions_block: Optional[dict], - scope_desc: str, - ) -> None: - """Add a rule from a permissions:{action} block.""" - if permissions_block is None: - return - - action_allow_block = permissions_block.get(self.action) - result = self.evaluate_allow_block(action_allow_block) - - self.collector.add( - parent=parent, - child=child, - allow=result, - reason=f"config {'allow' if result else 'deny'} {scope_desc}", - if_not_none=True, - ) - - def add_allow_block_rule( - self, - parent: Optional[str], - child: Optional[str], - allow_block: Any, - scope_desc: str, - ) -> None: - """ - Add rules from an allow:{} block. - - For allow blocks, if the block exists but doesn't match the actor, - this is treated as a deny. We also handle the restriction-gate logic. - """ - if allow_block is None: - return - - # Skip if resource is not in restriction allowlist - if not self.is_in_restriction_allowlist(parent, child): - return - - result = self.evaluate_allow_block(allow_block) - bool_result = bool(result) - - self.collector.add( - parent, - child, - bool_result, - f"config {'allow' if result else 'deny'} {scope_desc}", - ) - - # Handle restriction-gate: add explicit denies for restricted resources - self._add_restriction_gate_denies(parent, child, bool_result, scope_desc) - - def _add_restriction_gate_denies( - self, - parent: Optional[str], - child: Optional[str], - is_allowed: bool, - scope_desc: str, - ) -> None: - """ - When a config rule denies at a higher level, add explicit denies - for restricted resources to prevent child-level allows from - incorrectly granting access. - """ - if is_allowed or child is not None or not self.has_restrictions: - return - - if not self.action_obj: - return - - reason = f"config deny {scope_desc} (restriction gate)" - - if parent is None: - # Root-level deny: add denies for all restricted resources - if self.action_obj.takes_parent: - for db_name in self.restricted_databases: - self.collector.add(db_name, None, False, reason) - if self.action_obj.takes_child: - for db_name, table_name in self.restricted_tables: - self.collector.add(db_name, table_name, False, reason) - else: - # Database-level deny: add denies for tables in that database - if self.action_obj.takes_child: - for db_name, table_name in self.restricted_tables: - if db_name == parent: - self.collector.add(db_name, table_name, False, reason) - - def process(self) -> Optional[PermissionSQL]: - """Process all config rules and return combined PermissionSQL.""" - self._process_root_permissions() - self._process_databases() - self._process_root_allow_blocks() - - return self.collector.to_permission_sql() - - def _process_root_permissions(self) -> None: - """Process root-level permissions block.""" - root_perms = self.config.get("permissions") or {} - self.add_permissions_rule( - None, - None, - root_perms, - f"permissions for {self.action}", - ) - - def _process_databases(self) -> None: - """Process database-level and nested configurations.""" - databases = self.config.get("databases") or {} - - for db_name, db_config in databases.items(): - self._process_database(db_name, db_config or {}) - - def _process_database(self, db_name: str, db_config: dict) -> None: - """Process a single database's configuration.""" - # Database-level permissions block - db_perms = db_config.get("permissions") or {} - self.add_permissions_rule( - db_name, - None, - db_perms, - f"permissions for {self.action} on {db_name}", - ) - - # Process tables - for table_name, table_config in (db_config.get("tables") or {}).items(): - self._process_table(db_name, table_name, table_config or {}) - - # Process queries - for query_name, query_config in (db_config.get("queries") or {}).items(): - self._process_query(db_name, query_name, query_config) - - # Database-level allow blocks - self._process_database_allow_blocks(db_name, db_config) - - def _process_table( - self, - db_name: str, - table_name: str, - table_config: dict, - ) -> None: - """Process a single table's configuration.""" - # Table-level permissions block - table_perms = table_config.get("permissions") or {} - self.add_permissions_rule( - db_name, - table_name, - table_perms, - f"permissions for {self.action} on {db_name}/{table_name}", - ) - - # Table-level allow block (for view-table) - if self.action == "view-table": - self.add_allow_block_rule( - db_name, - table_name, - table_config.get("allow"), - f"allow for {self.action} on {db_name}/{table_name}", - ) - - def _process_query( - self, - db_name: str, - query_name: str, - query_config: Any, - ) -> None: - """Process a single query's configuration.""" - # Query config can be a string (just SQL) or dict - if not isinstance(query_config, dict): - return - - # Query-level permissions block - query_perms = query_config.get("permissions") or {} - self.add_permissions_rule( - db_name, - query_name, - query_perms, - f"permissions for {self.action} on {db_name}/{query_name}", - ) - - # Query-level allow block (for view-query) - if self.action == "view-query": - self.add_allow_block_rule( - db_name, - query_name, - query_config.get("allow"), - f"allow for {self.action} on {db_name}/{query_name}", - ) - - def _process_database_allow_blocks( - self, - db_name: str, - db_config: dict, - ) -> None: - """Process database-level allow/allow_sql blocks.""" - # view-database allow block - if self.action == "view-database": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow"), - f"allow for {self.action} on {db_name}", - ) - - # execute-sql allow_sql block - if self.action == "execute-sql": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow_sql"), - f"allow_sql for {db_name}", - ) - - # view-table uses database-level allow for inheritance - if self.action == "view-table": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow"), - f"allow for {self.action} on {db_name}", - ) - - # view-query uses database-level allow for inheritance - if self.action == "view-query": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow"), - f"allow for {self.action} on {db_name}", - ) - - def _process_root_allow_blocks(self) -> None: - """Process root-level allow/allow_sql blocks.""" - root_allow = self.config.get("allow") - - if self.action == "view-instance": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-instance", - ) - - if self.action == "view-database": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-database", - ) - - if self.action == "view-table": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-table", - ) - - if self.action == "view-query": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-query", - ) - - if self.action == "execute-sql": - self.add_allow_block_rule( - None, - None, - self.config.get("allow_sql"), - "allow_sql", - ) - - -@hookimpl(specname="permission_resources_sql") -async def config_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[List[PermissionSQL]]: - """ - Apply permission rules from datasette.yaml configuration. - - This processes: - - permissions: blocks at root, database, table, and query levels - - allow: blocks for view-* actions - - allow_sql: blocks for execute-sql action - """ - processor = ConfigPermissionProcessor(datasette, actor, action) - result = processor.process() - - if result is None: - return [] - - return [result] diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py deleted file mode 100644 index 5bc74425..00000000 --- a/datasette/default_permissions/defaults.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -Default permission settings for Datasette. - -Provides default allow rules for standard view/execute actions. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - -# Actions that are allowed by default (unless --default-deny is used) -DEFAULT_ALLOW_ACTIONS = frozenset( - { - "view-instance", - "view-database", - "view-database-download", - "view-table", - "view-query", - "execute-sql", - } -) - - -@hookimpl(specname="permission_resources_sql") -async def default_allow_sql_check( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - """ - Enforce the default_allow_sql setting. - - When default_allow_sql is false (the default), execute-sql is denied - unless explicitly allowed by config or other rules. - """ - if action == "execute-sql": - if not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") - - return None - - -@hookimpl(specname="permission_resources_sql") -async def default_action_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - """ - Provide default allow rules for standard view/execute actions. - - These defaults are skipped when datasette is started with --default-deny. - The restriction_sql mechanism (from actor_restrictions_sql) will still - filter these results if the actor has restrictions. - """ - if datasette.default_deny: - return None - - if action in DEFAULT_ALLOW_ACTIONS: - reason = f"default allow for {action}".replace("'", "''") - return PermissionSQL.allow(reason=reason) - - return None - - -@hookimpl(specname="permission_resources_sql") -async def default_query_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - actor_id = actor.get("id") if isinstance(actor, dict) else None - - if action not in {"view-query", "update-query", "delete-query"}: - return None - - params = {"query_owner_id": actor_id} - rule_sqls = [] - if actor_id is not None: - if action in {"update-query", "delete-query"}: - # Query owner can update/delete query - rule_sqls.append(""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE source = 'user' - AND owner_id = :query_owner_id - """) - else: - # Query owner can view-query - rule_sqls.append(""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE owner_id = :query_owner_id - """) - - # restriction_sql enforces private queries ONLY visible/mutable by owner - return PermissionSQL( - sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, - restriction_sql=""" - SELECT database_name AS parent, name AS child - FROM queries - WHERE is_private = 0 - OR owner_id = :query_owner_id - """, - params=params, - ) diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py deleted file mode 100644 index 47e03569..00000000 --- a/datasette/default_permissions/helpers.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Shared helper utilities for default permission implementations. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette.permissions import PermissionSQL - - -def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: - """ - Get all name variants for an action (full name and abbreviation). - - Example: - get_action_name_variants(ds, "view-table") -> {"view-table", "vt"} - """ - variants = {action} - action_obj = datasette.actions.get(action) - if action_obj and action_obj.abbr: - variants.add(action_obj.abbr) - return variants - - -def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: - """Check if an action (or its abbreviation) is in a list.""" - return bool(get_action_name_variants(datasette, action).intersection(action_list)) - - -@dataclass -class PermissionRow: - """A single permission rule row.""" - - parent: Optional[str] - child: Optional[str] - allow: bool - reason: str - - -class PermissionRowCollector: - """Collects permission rows and converts them to PermissionSQL.""" - - def __init__(self, prefix: str = "row"): - self.rows: List[PermissionRow] = [] - self.prefix = prefix - - def add( - self, - parent: Optional[str], - child: Optional[str], - allow: Optional[bool], - reason: str, - if_not_none: bool = False, - ) -> None: - """Add a permission row. If if_not_none=True, only add if allow is not None.""" - if if_not_none and allow is None: - return - self.rows.append(PermissionRow(parent, child, allow, reason)) - - def to_permission_sql(self) -> Optional[PermissionSQL]: - """Convert collected rows to a PermissionSQL object.""" - if not self.rows: - return None - - parts = [] - params = {} - - for idx, row in enumerate(self.rows): - key = f"{self.prefix}_{idx}" - parts.append( - f"SELECT :{key}_parent AS parent, :{key}_child AS child, " - f":{key}_allow AS allow, :{key}_reason AS reason" - ) - params[f"{key}_parent"] = row.parent - params[f"{key}_child"] = row.child - params[f"{key}_allow"] = 1 if row.allow else 0 - params[f"{key}_reason"] = row.reason - - sql = "\nUNION ALL\n".join(parts) - return PermissionSQL(sql=sql, params=params) diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py deleted file mode 100644 index a22cd7e5..00000000 --- a/datasette/default_permissions/restrictions.py +++ /dev/null @@ -1,195 +0,0 @@ -""" -Actor restriction handling for Datasette permissions. - -This module handles the _r (restrictions) key in actor dictionaries, which -contains allowlists of resources the actor can access. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set, Tuple - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - -from .helpers import action_in_list, get_action_name_variants - - -@dataclass -class ActorRestrictions: - """Parsed actor restrictions from the _r key.""" - - global_actions: List[str] # _r.a - globally allowed actions - database_actions: dict # _r.d - {db_name: [actions]} - table_actions: dict # _r.r - {db_name: {table: [actions]}} - - @classmethod - def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: - """Parse restrictions from actor dict. Returns None if no restrictions.""" - if not actor: - return None - assert isinstance(actor, dict), "actor must be a dictionary" - - restrictions = actor.get("_r") - if restrictions is None: - return None - - return cls( - global_actions=restrictions.get("a", []), - database_actions=restrictions.get("d", {}), - table_actions=restrictions.get("r", {}), - ) - - def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: - """Check if action is in the global allowlist.""" - return action_in_list(datasette, action, self.global_actions) - - def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: - """Get database names where this action is allowed.""" - allowed = set() - for db_name, db_actions in self.database_actions.items(): - if action_in_list(datasette, action, db_actions): - allowed.add(db_name) - return allowed - - def get_allowed_tables( - self, datasette: "Datasette", action: str - ) -> Set[Tuple[str, str]]: - """Get (database, table) pairs where this action is allowed.""" - allowed = set() - for db_name, tables in self.table_actions.items(): - for table_name, table_actions in tables.items(): - if action_in_list(datasette, action, table_actions): - allowed.add((db_name, table_name)) - return allowed - - -@hookimpl(specname="permission_resources_sql") -async def actor_restrictions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[List[PermissionSQL]]: - """ - Handle actor restriction-based permission rules. - - When an actor has an "_r" key, it contains an allowlist of resources they - can access. This function returns restriction_sql that filters the final - results to only include resources in that allowlist. - - The _r structure: - { - "a": ["vi", "pd"], # Global actions allowed - "d": {"mydb": ["vt", "es"]}, # Database-level actions - "r": {"mydb": {"users": ["vt"]}} # Table-level actions - } - """ - if not actor: - return None - - restrictions = ActorRestrictions.from_actor(actor) - - if restrictions is None: - # No restrictions - all resources allowed - return [] - - # If globally allowed, no filtering needed - if restrictions.is_action_globally_allowed(datasette, action): - return [] - - # Build restriction SQL - allowed_dbs = restrictions.get_allowed_databases(datasette, action) - allowed_tables = restrictions.get_allowed_tables(datasette, action) - - # If nothing is allowed for this action, return empty-set restriction - if not allowed_dbs and not allowed_tables: - return [ - PermissionSQL( - params={"deny": f"actor restrictions: {action} not in allowlist"}, - restriction_sql="SELECT NULL AS parent, NULL AS child WHERE 0", - ) - ] - - # Build UNION of allowed resources - selects = [] - params = {} - counter = 0 - - # Database-level entries (parent, NULL) - allows all children - for db_name in allowed_dbs: - key = f"restr_{counter}" - counter += 1 - selects.append(f"SELECT :{key}_parent AS parent, NULL AS child") - params[f"{key}_parent"] = db_name - - # Table-level entries (parent, child) - for db_name, table_name in allowed_tables: - key = f"restr_{counter}" - counter += 1 - selects.append(f"SELECT :{key}_parent AS parent, :{key}_child AS child") - params[f"{key}_parent"] = db_name - params[f"{key}_child"] = table_name - - restriction_sql = "\nUNION ALL\n".join(selects) - - return [PermissionSQL(params=params, restriction_sql=restriction_sql)] - - -def restrictions_allow_action( - datasette: "Datasette", - restrictions: dict, - action: str, - resource: Optional[str | Tuple[str, str]], -) -> bool: - """ - Check if restrictions allow the requested action on the requested resource. - - This is a synchronous utility function for use by other code that needs - to quickly check restriction allowlists. - - Args: - datasette: The Datasette instance - restrictions: The _r dict from an actor - action: The action name to check - resource: None for global, str for database, (db, table) tuple for table - - Returns: - True if allowed, False if denied - """ - # Does this action have an abbreviation? - to_check = get_action_name_variants(datasette, action) - - # Check global level (any resource) - all_allowed = restrictions.get("a") - if all_allowed is not None: - assert isinstance(all_allowed, list) - if to_check.intersection(all_allowed): - return True - - # Check database level - if resource: - if isinstance(resource, str): - database_name = resource - else: - database_name = resource[0] - database_allowed = restrictions.get("d", {}).get(database_name) - if database_allowed is not None: - assert isinstance(database_allowed, list) - if to_check.intersection(database_allowed): - return True - - # Check table/resource level - if resource is not None and not isinstance(resource, str) and len(resource) == 2: - database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True - - # This action is not explicitly allowed, so reject it - return False diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py deleted file mode 100644 index 4931f7ff..00000000 --- a/datasette/default_permissions/root.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Root user permission handling for Datasette. - -Grants full permissions to the root user when --root flag is used. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - - -@hookimpl(specname="permission_resources_sql") -async def root_user_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], -) -> Optional[PermissionSQL]: - """ - Grant root user full permissions when --root flag is used. - """ - if not datasette.root_enabled: - return None - if actor is not None and actor.get("id") == "root": - return PermissionSQL.allow(reason="root user") diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py deleted file mode 100644 index 7a359dc6..00000000 --- a/datasette/default_permissions/tokens.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Token authentication for Datasette. - -Registers the default SignedTokenHandler and delegates token verification -to datasette.verify_token() so all registered handlers are tried. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.tokens import SignedTokenHandler - - -@hookimpl -def register_token_handler(datasette: "Datasette"): - """Register the default signed token handler.""" - return SignedTokenHandler() - - -@hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token( - datasette: "Datasette", request -) -> Optional[dict]: - """ - Authenticate requests using API tokens by delegating to all registered - token handlers via datasette.verify_token(). - """ - authorization = request.headers.get("authorization") - if not authorization: - return None - if not authorization.startswith("Bearer "): - return None - - token = authorization[len("Bearer ") :] - return await datasette.verify_token(token) diff --git a/datasette/events.py b/datasette/events.py deleted file mode 100644 index e8786da9..00000000 --- a/datasette/events.py +++ /dev/null @@ -1,293 +0,0 @@ -from abc import ABC, abstractproperty -from dataclasses import asdict, dataclass, field -from datasette.hookspecs import hookimpl -from datetime import datetime, timezone - - -@dataclass -class Event(ABC): - @abstractproperty - def name(self): - pass - - created: datetime = field( - init=False, default_factory=lambda: datetime.now(timezone.utc) - ) - actor: dict | None - - def properties(self): - properties = asdict(self) - properties.pop("actor", None) - properties.pop("created", None) - return properties - - -@dataclass -class LoginEvent(Event): - """ - Event name: ``login`` - - A user (represented by ``event.actor``) has logged in. - """ - - name = "login" - - -@dataclass -class LogoutEvent(Event): - """ - Event name: ``logout`` - - A user (represented by ``event.actor``) has logged out. - """ - - name = "logout" - - -@dataclass -class CreateTokenEvent(Event): - """ - Event name: ``create-token`` - - A user created an API token. - - :ivar expires_after: Number of seconds after which this token will expire. - :type expires_after: int or None - :ivar restrict_all: Restricted permissions for this token. - :type restrict_all: list - :ivar restrict_database: Restricted database permissions for this token. - :type restrict_database: dict - :ivar restrict_resource: Restricted resource permissions for this token. - :type restrict_resource: dict - """ - - name = "create-token" - expires_after: int | None - restrict_all: list - restrict_database: dict - restrict_resource: dict - - -@dataclass -class CreateTableEvent(Event): - """ - Event name: ``create-table`` - - A new table has been created in the database. - - :ivar database: The name of the database where the table was created. - :type database: str - :ivar table: The name of the table that was created - :type table: str - :ivar schema: The SQL schema definition for the new table. - :type schema: str - """ - - name = "create-table" - database: str - table: str - schema: str - - -@dataclass -class DropTableEvent(Event): - """ - Event name: ``drop-table`` - - A table has been dropped from the database. - - :ivar database: The name of the database where the table was dropped. - :type database: str - :ivar table: The name of the table that was dropped - :type table: str - """ - - name = "drop-table" - database: str - table: str - - -@dataclass -class AlterTableEvent(Event): - """ - Event name: ``alter-table`` - - A table has been altered. - - :ivar database: The name of the database where the table was altered - :type database: str - :ivar table: The name of the table that was altered - :type table: str - :ivar before_schema: The table's SQL schema before the alteration - :type before_schema: str - :ivar after_schema: The table's SQL schema after the alteration - :type after_schema: str - """ - - name = "alter-table" - database: str - table: str - before_schema: str - after_schema: str - - -@dataclass -class InsertRowsEvent(Event): - """ - Event name: ``insert-rows`` - - Rows were inserted into a table. - - :ivar database: The name of the database where the rows were inserted. - :type database: str - :ivar table: The name of the table where the rows were inserted. - :type table: str - :ivar num_rows: The number of rows that were requested to be inserted. - :type num_rows: int - :ivar ignore: Was ignore set? - :type ignore: bool - :ivar replace: Was replace set? - :type replace: bool - """ - - name = "insert-rows" - database: str - table: str - num_rows: int - ignore: bool - replace: bool - - -@dataclass -class UpsertRowsEvent(Event): - """ - Event name: ``upsert-rows`` - - Rows were upserted into a table. - - :ivar database: The name of the database where the rows were inserted. - :type database: str - :ivar table: The name of the table where the rows were inserted. - :type table: str - :ivar num_rows: The number of rows that were requested to be inserted. - :type num_rows: int - """ - - name = "upsert-rows" - database: str - table: str - num_rows: int - - -@dataclass -class UpdateRowEvent(Event): - """ - Event name: ``update-row`` - - A row was updated in a table. - - :ivar database: The name of the database where the row was updated. - :type database: str - :ivar table: The name of the table where the row was updated. - :type table: str - :ivar pks: The primary key values of the updated row. - """ - - name = "update-row" - database: str - table: str - pks: list - - -@dataclass -class RenameTableEvent(Event): - """ - Event name: ``rename-table`` - - A table has been renamed. - - :ivar database: The name of the database containing the renamed table. - :type database: str - :ivar old_table: The previous name of the table. - :type old_table: str - :ivar new_table: The new name of the table. - :type new_table: str - """ - - name = "rename-table" - database: str - old_table: str - new_table: str - - -@dataclass -class DeleteRowEvent(Event): - """ - Event name: ``delete-row`` - - A row was deleted from a table. - - :ivar database: The name of the database where the row was deleted. - :type database: str - :ivar table: The name of the table where the row was deleted. - :type table: str - :ivar pks: The primary key values of the deleted row. - """ - - name = "delete-row" - database: str - table: str - pks: list - - -@hookimpl -def write_wrapper(datasette, database, request, transaction): - def wrapper(conn, track_event): - # Snapshot rootpage -> name before the write - before = { - row[1]: row[0] - for row in conn.execute( - "select name, rootpage from sqlite_master" - " where type='table' and rootpage != 0" - ).fetchall() - } - yield - # Snapshot rootpage -> name after the write - after = { - row[1]: row[0] - for row in conn.execute( - "select name, rootpage from sqlite_master" - " where type='table' and rootpage != 0" - ).fetchall() - } - # Detect renames: same rootpage, different name - for rootpage, old_name in before.items(): - new_name = after.get(rootpage) - if new_name and new_name != old_name: - track_event( - RenameTableEvent( - actor=request.actor if request else None, - database=database, - old_table=old_name, - new_table=new_name, - ) - ) - - return wrapper - - -@hookimpl -def register_events(): - return [ - LoginEvent, - LogoutEvent, - CreateTableEvent, - CreateTokenEvent, - AlterTableEvent, - RenameTableEvent, - DropTableEvent, - InsertRowsEvent, - UpsertRowsEvent, - UpdateRowEvent, - DeleteRowEvent, - ] diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..b23615fe 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -11,8 +11,8 @@ from datasette.utils import ( ) -def load_facet_configs(request, table_config): - # Given a request and the configuration for a table, return +def load_facet_configs(request, table_metadata): + # Given a request and the metadata configuration for a table, return # a dictionary of selected facets, their lists of configs and for each # config whether it came from the request or the metadata. # @@ -20,21 +20,21 @@ def load_facet_configs(request, table_config): # {"source": "metadata", "config": config1}, # {"source": "request", "config": config2}]} facet_configs = {} - table_config = table_config or {} - table_facet_configs = table_config.get("facets", []) - for facet_config in table_facet_configs: - if isinstance(facet_config, str): + table_metadata = table_metadata or {} + metadata_facets = table_metadata.get("facets", []) + for metadata_config in metadata_facets: + if isinstance(metadata_config, str): type = "column" - facet_config = {"simple": facet_config} + metadata_config = {"simple": metadata_config} else: assert ( - len(facet_config.values()) == 1 + len(metadata_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, facet_config = list(facet_config.items())[0] - if isinstance(facet_config, str): - facet_config = {"simple": facet_config} + type, metadata_config = list(metadata_config.items())[0] + if isinstance(metadata_config, str): + metadata_config = {"simple": metadata_config} facet_configs.setdefault(type, []).append( - {"source": "metadata", "config": facet_config} + {"source": "metadata", "config": metadata_config} ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): @@ -45,12 +45,13 @@ def load_facet_configs(request, table_config): elif key.startswith("_facet_"): type = key[len("_facet_") :] for value in values: - # The value is the facet_config - either JSON or not - facet_config = ( - json.loads(value) if value.startswith("{") else {"simple": value} - ) + # The value is the config - either JSON or not + if value.startswith("{"): + config = json.loads(value) + else: + config = {"simple": value} facet_configs.setdefault(type, []).append( - {"source": "request", "config": facet_config} + {"source": "request", "config": config} ) return facet_configs @@ -65,8 +66,6 @@ def register_facet_classes(): class Facet: type = None - # How many rows to consider when suggesting facets: - suggest_consider = 1000 def __init__( self, @@ -76,23 +75,23 @@ class Facet: sql=None, table=None, params=None, - table_config=None, + metadata=None, row_count=None, ): assert table or sql, "Must provide either table= or sql=" self.ds = ds self.request = request self.database = database - # For foreign key expansion. Can be None for e.g. stored SQL queries: + # For foreign key expansion. Can be None for e.g. canned SQL queries: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] - self.table_config = table_config + self.metadata = metadata # row_count can be None, in which case we calculate it ourselves: self.row_count = row_count def get_configs(self): - configs = load_facet_configs(self.request, self.table_config) + configs = load_facet_configs(self.request, self.metadata) return configs.get(self.type) or [] def get_querystring_pairs(self): @@ -105,15 +104,10 @@ class Facet: max_returned_rows = self.ds.setting("max_returned_rows") table_facet_size = None if self.table: - config_facet_size = ( - self.ds.config.get("databases", {}) - .get(self.database, {}) - .get("tables", {}) - .get(self.table, {}) - .get("facet_size") - ) - if config_facet_size: - table_facet_size = config_facet_size + tables_metadata = self.ds.metadata("tables", database=self.database) or {} + table_metadata = tables_metadata.get(self.table) or {} + if table_metadata: + table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") if custom_facet_size: if custom_facet_size == "max": @@ -147,6 +141,17 @@ class Facet: ) ).columns + async def get_row_count(self): + if self.row_count is None: + self.row_count = ( + await self.ds.execute( + self.database, + f"select count(*) from ({self.sql})", + self.params, + ) + ).rows[0][0] + return self.row_count + class ColumnFacet(Facet): type = "column" @@ -161,16 +166,13 @@ class ColumnFacet(Facet): if column in already_enabled: continue suggested_facet_sql = """ - with limited as (select * from ({sql}) limit {suggest_consider}) - select {column} as value, count(*) as n from limited - where value is not null + select {column} as value, count(*) as n from ( + {sql} + ) where value is not null group by value limit {limit} """.format( - column=escape_sqlite(column), - sql=self.sql, - limit=facet_size + 1, - suggest_consider=self.suggest_consider, + column=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 ) distinct_values = None try: @@ -205,17 +207,6 @@ class ColumnFacet(Facet): continue return suggested_facets - async def get_row_count(self): - if self.row_count is None: - self.row_count = ( - await self.ds.execute( - self.database, - f"select count(*) from (select * from ({self.sql}) limit {self.suggest_consider})", - self.params, - ) - ).rows[0][0] - return self.row_count - async def facet_results(self): facet_results = [] facets_timed_out = [] @@ -233,7 +224,9 @@ class ColumnFacet(Facet): ) where {col} is not null group by {col} order by count desc, value limit {limit} - """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1) + """.format( + col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 + ) try: facet_rows_results = await self.ds.execute( self.database, @@ -316,14 +309,11 @@ class ArrayFacet(Facet): continue # Is every value in this column either null or a JSON array? suggested_facet_sql = """ - with limited as (select * from ({sql}) limit {suggest_consider}) select distinct json_type({column}) - from limited + from ({sql}) where {column} is not null and {column} != '' """.format( - column=escape_sqlite(column), - sql=self.sql, - suggest_consider=self.suggest_consider, + column=escape_sqlite(column), sql=self.sql ) try: results = await self.ds.execute( @@ -408,9 +398,7 @@ class ArrayFacet(Facet): order by count(*) desc, value limit {limit} """.format( - col=escape_sqlite(column), - sql=self.sql, - limit=facet_size + 1, + col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 ) try: facet_rows_results = await self.ds.execute( @@ -478,9 +466,11 @@ class DateFacet(Facet): # Does this column contain any dates in the first 100 rows? suggested_facet_sql = """ select date({column}) from ( - select * from ({sql}) limit 100 - ) where {column} glob "????-??-*" - """.format(column=escape_sqlite(column), sql=self.sql) + {sql} + ) where {column} glob "????-??-*" limit 100; + """.format( + column=escape_sqlite(column), sql=self.sql + ) try: results = await self.ds.execute( self.database, @@ -526,7 +516,9 @@ class DateFacet(Facet): ) where date({col}) is not null group by date({col}) order by count desc, value limit {limit} - """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1) + """.format( + col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 + ) try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 95cc5f37..5ea3488b 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,8 +1,8 @@ from datasette import hookimpl -from datasette.resources import DatabaseResource from datasette.views.base import DatasetteError from datasette.utils.asgi import BadRequest import json +import numbers from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -13,10 +13,11 @@ def where_filters(request, database, datasette): where_clauses = [] extra_wheres_for_ui = [] if "_where" in request.args: - if not await datasette.allowed( - action="execute-sql", - resource=DatabaseResource(database=database), - actor=request.actor, + if not await datasette.permission_allowed( + request.actor, + "execute-sql", + resource=database, + default=True, ): raise DatasetteError("_where= is not allowed", status=403) else: @@ -49,7 +50,7 @@ def search_filters(request, database, table, datasette): extra_context = {} # Figure out which fts_table to use - table_metadata = await datasette.table_config(database, table) + table_metadata = datasette.table_metadata(database, table) db = datasette.get_database(database) fts_table = request.args.get("_fts_table") fts_table = fts_table or table_metadata.get("fts_table") @@ -79,9 +80,9 @@ def search_filters(request, database, table, datasette): "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( fts_table=escape_sqlite(fts_table), fts_pk=escape_sqlite(fts_pk), - match_clause=( - ":search" if search_mode_raw else "escape_fts(:search)" - ), + match_clause=":search" + if search_mode_raw + else "escape_fts(:search)", ) ) human_descriptions.append(f'search matches "{search}"') @@ -98,11 +99,9 @@ def search_filters(request, database, table, datasette): "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), - match_clause=( - ":search_{}".format(i) - if search_mode_raw - else "escape_fts(:search_{})".format(i) - ), + match_clause=":search_{}".format(i) + if search_mode_raw + else "escape_fts(:search_{})".format(i), ) ) human_descriptions.append( @@ -280,13 +279,6 @@ class Filters: '{c} contains "{v}"', format="%{}%", ), - TemplatedFilter( - "notcontains", - "does not contain", - '"{c}" not like :{p}', - '{c} does not contain "{v}"', - format="%{}%", - ), TemplatedFilter( "endswith", "ends with", @@ -367,8 +359,12 @@ class Filters: ) _filters_by_key = {f.key: f for f in _filters} - def __init__(self, pairs): + def __init__(self, pairs, units=None, ureg=None): + if units is None: + units = {} self.pairs = pairs + self.units = units + self.ureg = ureg def lookups(self): """Yields (lookup, display, no_argument) pairs""" @@ -408,6 +404,20 @@ class Filters: def has_selections(self): return bool(self.pairs) + def convert_unit(self, column, value): + """If the user has provided a unit in the query, convert it into the column unit, if present.""" + if column not in self.units: + return value + + # Try to interpret the value as a unit + value = self.ureg(value) + if isinstance(value, numbers.Number): + # It's just a bare number, assume it's the column unit + return value + + column_unit = self.ureg(self.units[column]) + return value.to(column_unit).magnitude + def build_where_clauses(self, table): sql_bits = [] params = {} @@ -415,7 +425,9 @@ class Filters: for column, lookup, value in self.selections(): filter = self._filters_by_key.get(lookup, None) if filter: - sql_bit, param = filter.where_clause(table, column, value, i) + sql_bit, param = filter.where_clause( + table, column, self.convert_unit(column, value), i + ) sql_bits.append(sql_bit) if param is not None: if not isinstance(param, list): diff --git a/datasette/fixtures.py b/datasette/fixtures.py deleted file mode 100644 index 7c85e16a..00000000 --- a/datasette/fixtures.py +++ /dev/null @@ -1,415 +0,0 @@ -from datasette.utils.sqlite import sqlite3 -from datasette.utils import documented -import itertools -import random -import string - -__all__ = [ - "EXTRA_DATABASE_SQL", - "TABLES", - "TABLE_PARAMETERIZED_SQL", - "generate_compound_rows", - "generate_sortable_rows", - "populate_extra_database", - "populate_fixture_database", - "write_extra_database", - "write_fixture_database", -] - - -def generate_compound_rows(num): - """Generate rows for the compound_three_primary_keys fixture table.""" - for a, b, c in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=3), num - ): - yield a, b, c, f"{a}-{b}-{c}" - - -def generate_sortable_rows(num): - """Generate rows for the sortable fixture table.""" - rand = random.Random(42) - for a, b in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=2), num - ): - yield { - "pk1": a, - "pk2": b, - "content": f"{a}-{b}", - "sortable": rand.randint(-100, 100), - "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), - "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), - "text": rand.choice(["$null", "$blah"]), - } - - -TABLES = ( - """ -CREATE TABLE simple_primary_key ( - id integer primary key, - content text -); - -CREATE TABLE primary_key_multiple_columns ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE primary_key_multiple_columns_explicit_label ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE compound_primary_key ( - pk1 varchar(30), - pk2 varchar(30), - content text, - PRIMARY KEY (pk1, pk2) -); - -INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); -INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); -INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); - -CREATE TABLE compound_three_primary_keys ( - pk1 varchar(30), - pk2 varchar(30), - pk3 varchar(30), - content text, - PRIMARY KEY (pk1, pk2, pk3) -); -CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); - -CREATE TABLE foreign_key_references ( - pk varchar(30) primary key, - foreign_key_with_label integer, - foreign_key_with_blank_label integer, - foreign_key_with_no_label varchar(30), - foreign_key_compound_pk1 varchar(30), - foreign_key_compound_pk2 varchar(30), - FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) - FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) -); - -CREATE TABLE sortable ( - pk1 varchar(30), - pk2 varchar(30), - content text, - sortable integer, - sortable_with_nulls real, - sortable_with_nulls_2 real, - text text, - PRIMARY KEY (pk1, pk2) -); - -CREATE TABLE no_primary_key ( - content text, - a text, - b text, - c text -); - -CREATE TABLE [123_starts_with_digits] ( - content text -); - -CREATE VIEW paginated_view AS - SELECT - content, - '- ' || content || ' -' AS content_extra - FROM no_primary_key; - -CREATE TABLE "Table With Space In Name" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "table/with/slashes.csv" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "complex_foreign_keys" ( - pk varchar(30) primary key, - f1 integer, - f2 integer, - f3 integer, - FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) -); - -CREATE TABLE "custom_foreign_key_label" ( - pk varchar(30) primary key, - foreign_key_with_custom_label text, - FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) -); - -CREATE TABLE tags ( - tag TEXT PRIMARY KEY -); - -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text, - [name with . and spaces] text -); - -CREATE TABLE searchable_tags ( - searchable_id integer, - tag text, - PRIMARY KEY (searchable_id, tag), - FOREIGN KEY (searchable_id) REFERENCES searchable(pk), - FOREIGN KEY (tag) REFERENCES tags(tag) -); - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); - -INSERT INTO tags VALUES ("canine"); -INSERT INTO tags VALUES ("feline"); - -INSERT INTO searchable_tags (searchable_id, tag) VALUES - (1, "feline"), - (2, "canine") -; - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); -INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); - -CREATE TABLE [select] ( - [group] text, - [having] text, - [and] text, - [json] text -); -INSERT INTO [select] VALUES ('group', 'having', 'and', - '{"href": "http://example.com/", "label":"Example"}' -); - -CREATE TABLE infinity ( - value REAL -); -INSERT INTO infinity VALUES - (1e999), - (-1e999), - (1.5) -; - -CREATE TABLE facet_cities ( - id integer primary key, - name text -); -INSERT INTO facet_cities (id, name) VALUES - (1, 'San Francisco'), - (2, 'Los Angeles'), - (3, 'Detroit'), - (4, 'Memnonia') -; - -CREATE TABLE facetable ( - pk integer primary key, - created text, - planet_int integer, - on_earth integer, - state text, - _city_id integer, - _neighborhood text, - tags text, - complex_array text, - distinct_some_null, - n text, - FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) -); -INSERT INTO facetable - (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) -VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) -; - -CREATE TABLE binary_data ( - data BLOB -); - --- Many 2 Many demo: roadside attractions! - -CREATE TABLE roadside_attractions ( - pk integer primary key, - name text, - address text, - url text, - latitude real, - longitude real -); -INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", - 37.0167, -122.0024 -); -INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", - 37.3184, -121.9511 -); -INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, - 37.5793, -122.3442 -); -INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", - 37.0414, -122.0725 -); - -CREATE TABLE attraction_characteristic ( - pk integer primary key, - name text -); -INSERT INTO attraction_characteristic VALUES ( - 1, "Museum" -); -INSERT INTO attraction_characteristic VALUES ( - 2, "Paranormal" -); - -CREATE TABLE roadside_attraction_characteristics ( - attraction_id INTEGER REFERENCES roadside_attractions(pk), - characteristic_id INTEGER REFERENCES attraction_characteristic(pk) -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 1, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 2, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 3, 1 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 1 -); - -INSERT INTO simple_primary_key VALUES (1, 'hello'); -INSERT INTO simple_primary_key VALUES (2, 'world'); -INSERT INTO simple_primary_key VALUES (3, ''); -INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); -INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); - -INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); -INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); - -INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); -INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); - -INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); -INSERT INTO custom_foreign_key_label VALUES (1, 1); - -INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); - -CREATE VIEW simple_view AS - SELECT content, upper(content) AS upper_content FROM simple_primary_key; - -CREATE VIEW searchable_view AS - SELECT * from searchable; - -CREATE VIEW searchable_view_configured_by_metadata AS - SELECT * from searchable; - -""" - + "\n".join( - [ - 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( - i=i + 1 - ) - for i in range(201) - ] - ) - + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' - + "\n".join( - [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) - for a, b, c, content in generate_compound_rows(1001) - ] - ) - + "\n".join(["""INSERT INTO sortable VALUES ( - "{pk1}", "{pk2}", "{content}", {sortable}, - {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); - """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) -) - -TABLE_PARAMETERIZED_SQL = [ - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (null);", []), -] - -EXTRA_DATABASE_SQL = """ -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text -); - -CREATE VIEW searchable_view AS SELECT * FROM searchable; - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS3 (text1, text2, content="searchable"); -INSERT INTO "searchable_fts" (rowid, text1, text2) - SELECT rowid, text1, text2 FROM searchable; -""" - - -@documented(label="datasette_fixtures_populate_fixture_database") -def populate_fixture_database(conn): - """Populate a SQLite connection with Datasette's test fixture tables.""" - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - - -def populate_extra_database(conn): - """Populate a SQLite connection with the extra database used in tests.""" - conn.executescript(EXTRA_DATABASE_SQL) - - -def write_fixture_database(db_filename): - """Write Datasette's test fixture tables to a SQLite database file.""" - conn = sqlite3.connect(db_filename) - try: - populate_fixture_database(conn) - finally: - conn.close() - - -def write_extra_database(db_filename): - """Write the extra test database tables to a SQLite database file.""" - conn = sqlite3.connect(db_filename) - try: - populate_extra_database(conn) - finally: - conn.close() diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 41c48396..156a44d4 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,3 +1,4 @@ +from os import stat from datasette import hookimpl, Response diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 96398a4c..8b7e83e3 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,16 +1,14 @@ from datasette import hookimpl, Response -from .utils import add_cors_headers +from .utils import await_me_maybe, add_cors_headers from .utils.asgi import ( Base400, + Forbidden, ) from .views.base import DatasetteError from markupsafe import Markup +import pdb import traceback - -try: - import ipdb as pdb -except ImportError: - import pdb +from .plugins import pm try: import rich @@ -59,8 +57,7 @@ def handle_exception(datasette, request, exception): if request.path.split("?")[0].endswith(".json"): return Response.json(info, status=status, headers=headers) else: - environment = datasette.get_jinja_environment(request) - template = environment.select_template(templates) + template = datasette.jinja_env.select_template(templates) return Response.html( await template.render_async( dict( diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index dcd502af..9069927b 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -10,6 +10,11 @@ def startup(datasette): """Fires directly after Datasette first starts running""" +@hookspec +def get_metadata(datasette, key, database, table): + """Return metadata to be merged into Datasette's metadata dictionary""" + + @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" @@ -55,17 +60,7 @@ def publish_subcommand(publish): @hookspec -def render_cell( - row, - value, - column, - table, - pks, - database, - datasette, - request, - column_type, -): +def render_cell(row, value, column, table, database, datasette, request): """Customize rendering of HTML table cell values""" @@ -80,13 +75,8 @@ def register_facet_classes(): @hookspec -def register_actions(datasette): - """Register actions: returns a list of datasette.permission.Action objects""" - - -@hookspec -def register_column_types(datasette): - """Return a list of ColumnType subclasses""" +def register_permissions(datasette): + """Register permissions: returns a list of datasette.permission.Permission named tuples""" @hookspec @@ -109,11 +99,6 @@ def actors_from_ids(datasette, actor_ids): """Returns a dictionary mapping those IDs to actor dictionaries""" -@hookspec -def jinja2_environment_from_request(datasette, request, env): - """Return a Jinja2 environment based on the incoming request""" - - @hookspec def filters_from_request(request, database, table, datasette): """ @@ -126,15 +111,13 @@ def filters_from_request(request, database, table, datasette): @hookspec -def permission_resources_sql(datasette, actor, action): - """Return SQL query fragments for permission checks on resources. +def permission_allowed(datasette, actor, action, resource): + """Check if actor is allowed to perform this action - return True, False or None""" - Returns None, a PermissionSQL object, or a list of PermissionSQL objects. - Each PermissionSQL contains SQL that should return rows with columns: - parent (str|None), child (str|None), allow (int), reason (str). - Used to efficiently check permissions across multiple resources at once. - """ +@hookspec +def canned_queries(datasette, database, actor): + """Return a dictionary of canned query definitions or an awaitable function that returns them""" @hookspec @@ -152,114 +135,21 @@ def menu_links(datasette, actor, request): """Links for the navigation menu""" -@hookspec -def jump_items_sql(datasette, actor, request): - """SQL fragments for extra items in the jump menu""" - - -@hookspec -def row_actions(datasette, actor, request, database, table, row): - """Links for the row actions menu""" - - @hookspec def table_actions(datasette, actor, database, table, request): """Links for the table actions menu""" -@hookspec -def view_actions(datasette, actor, database, view, request): - """Links for the view actions menu""" - - -@hookspec -def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and stored query actions menu""" - - @hookspec def database_actions(datasette, actor, database, request): """Links for the database actions menu""" @hookspec -def homepage_actions(datasette, actor, request): - """Links for the homepage actions menu""" +def skip_csrf(datasette, scope): + """Mechanism for skipping CSRF checks for certain requests""" @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" - - -@hookspec -def track_event(datasette, event): - """Respond to an event tracked by Datasette""" - - -@hookspec -def register_events(datasette): - """Return a list of Event subclasses to use with track_event()""" - - -@hookspec -def top_homepage(datasette, request): - """HTML to include at the top of the homepage""" - - -@hookspec -def top_database(datasette, request, database): - """HTML to include at the top of the database page""" - - -@hookspec -def top_table(datasette, request, database, table): - """HTML to include at the top of the table page""" - - -@hookspec -def top_row(datasette, request, database, table, row): - """HTML to include at the top of the row page""" - - -@hookspec -def top_query(datasette, request, database, sql): - """HTML to include at the top of the query results page""" - - -@hookspec -def top_stored_query(datasette, request, database, query_name): - """HTML to include at the top of the stored query page""" - - -@hookspec -def register_token_handler(datasette): - """Return a TokenHandler instance for token creation and verification""" - - -@hookspec -def write_wrapper(datasette, database, request, transaction): - """Called when a write function is about to execute. - - Return a generator function that accepts a ``conn`` argument and - optionally a ``track_event`` argument. The generator should - ``yield`` exactly once: code before the ``yield`` runs before - the write, code after the ``yield`` runs after the write - completes. The result of the write is sent back through the - ``yield``, so you can capture it with ``result = yield``. - - If your generator accepts ``track_event``, you can call - ``track_event(event)`` to queue an event that will be dispatched - via ``datasette.track_event()`` after the write commits - successfully. Events are discarded if the write raises an - exception. - - If the write raises an exception, it is thrown into the generator - so you can handle it with a try/except around the ``yield``. - - ``request`` may be ``None`` for writes not originating from an - HTTP request. ``transaction`` is ``True`` if the write will - be wrapped in a transaction. - - Return ``None`` to skip wrapping. - """ diff --git a/datasette/inspect.py b/datasette/inspect.py index 5e681e03..ede142d0 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -10,6 +10,7 @@ from .utils import ( sqlite3, ) + HASH_BLOCK_SIZE = 1024 * 1024 @@ -69,11 +70,16 @@ def inspect_tables(conn, database_metadata): tables[table]["foreign_keys"] = info # Mark tables 'hidden' if they relate to FTS virtual tables - hidden_tables = [r["name"] for r in conn.execute(""" + hidden_tables = [ + r["name"] + for r in conn.execute( + """ select name from sqlite_master where rootpage = 0 and sql like '%VIRTUAL TABLE%USING FTS%' - """)] + """ + ) + ] if detect_spatialite(conn): # Also hide Spatialite internal tables @@ -88,11 +94,14 @@ def inspect_tables(conn, database_metadata): "views_geometry_columns", "virts_geometry_columns", ] + [ - r["name"] for r in conn.execute(""" + r["name"] + for r in conn.execute( + """ select name from sqlite_master where name like "idx_%" and type = "table" - """) + """ + ) ] for t in tables.keys(): diff --git a/datasette/jump.py b/datasette/jump.py deleted file mode 100644 index d138e827..00000000 --- a/datasette/jump.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Any - - -@dataclass -class JumpSQL: - sql: str - params: dict[str, Any] | None = None - database: str | None = None - - @classmethod - def menu_item( - cls, - *, - label: str, - url: str, - description: str = "Menu item", - search_text: str | None = None, - display_name: str | None = None, - item_type: str = "menu", - ) -> "JumpSQL": - if search_text is None: - search_text = " ".join( - text for text in (label, display_name, description) if text is not None - ) - return cls( - sql=""" - SELECT - :type AS type, - :label AS label, - :description AS description, - :url AS url, - :search_text AS search_text, - :display_name AS display_name - """, - params={ - "type": item_type, - "label": label, - "description": description, - "url": url, - "search_text": search_text, - "display_name": display_name, - }, - ) - - -_PARAM_RE = re.compile(r"(? bool: - """ - Whether this resource is private (accessible to actor but not anonymous). - - This property is only available on Resource objects returned from - allowed_resources() when include_is_private=True is used. - - Raises: - AttributeError: If accessed without calling include_is_private=True - """ - if self._private is None: - raise AttributeError( - "The 'private' attribute is only available when using " - "allowed_resources(..., include_is_private=True)" - ) - return self._private - - @private.setter - def private(self, value: bool): - self._private = value - - @classmethod - def __init_subclass__(cls): - """ - Validate resource hierarchy doesn't exceed 2 levels. - - Raises: - ValueError: If this resource would create a 3-level hierarchy - """ - super().__init_subclass__() - - if cls.parent_class is None: - return # Top of hierarchy, nothing to validate - - # Check if our parent has a parent - that would create 3 levels - if cls.parent_class.parent_class is not None: - # We have a parent, and that parent has a parent - # This creates a 3-level hierarchy, which is not allowed - raise ValueError( - f"Resource {cls.__name__} creates a 3-level hierarchy: " - f"{cls.parent_class.parent_class.__name__} -> {cls.parent_class.__name__} -> {cls.__name__}. " - f"Maximum 2 levels allowed (parent -> child)." - ) - - @classmethod - @abstractmethod - async def resources_sql(cls, datasette, actor=None) -> str: - """ - Return SQL query that returns all resources of this type. - - Must return two columns: parent, child - """ - pass - - -class AllowedResource(NamedTuple): - """A resource with the reason it was allowed (for debugging).""" - - resource: Resource - reason: str - - -@dataclass(frozen=True, kw_only=True) -class Action: - name: str - description: str | None - abbr: str | None = None - resource_class: type[Resource] | None = None - also_requires: str | None = None # Optional action name that must also be allowed - - @property - def takes_parent(self) -> bool: - """ - Whether this action requires a parent identifier when instantiating its resource. - - Returns False for global-only actions (no resource_class). - Returns True for all actions with a resource_class (all resources require a parent identifier). - """ - return self.resource_class is not None - - @property - def takes_child(self) -> bool: - """ - Whether this action requires a child identifier when instantiating its resource. - - Returns False for global actions (no resource_class). - Returns False for parent-level resources (DatabaseResource - parent_class is None). - Returns True for child-level resources (TableResource, QueryResource - have a parent_class). - """ - if self.resource_class is None: - return False - return self.resource_class.parent_class is not None - - -_reason_id = 1 - - -@dataclass -class PermissionSQL: - """ - A plugin contributes SQL that yields: - parent TEXT NULL, - child TEXT NULL, - allow INTEGER, -- 1 allow, 0 deny - reason TEXT - - For restriction-only plugins, sql can be None and only restriction_sql is provided. - """ - - sql: str | None = ( - None # SQL that SELECTs the 4 columns above (can be None for restriction-only) - ) - params: dict[str, Any] | None = ( - None # bound params for the SQL (values only; no ':' prefix) - ) - source: str | None = None # System will set this to the plugin name - restriction_sql: str | None = ( - None # Optional SQL that returns (parent, child) for restriction filtering - ) - - @classmethod - def allow(cls, reason: str, _allow: bool = True) -> "PermissionSQL": - global _reason_id - i = _reason_id - _reason_id += 1 - return cls( - sql=f"SELECT NULL AS parent, NULL AS child, {1 if _allow else 0} AS allow, :reason_{i} AS reason", - params={f"reason_{i}": reason}, - ) - - @classmethod - def deny(cls, reason: str) -> "PermissionSQL": - return cls.allow(reason=reason, _allow=False) - - -# This is obsolete, replaced by Action and ResourceType @dataclass class Permission: name: str - abbr: str | None - description: str | None + abbr: Optional[str] + description: Optional[str] takes_database: bool takes_resource: bool default: bool diff --git a/datasette/plugins.py b/datasette/plugins.py index 5a31cdad..6ec08a81 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,20 +1,10 @@ import importlib import os import pluggy -from pprint import pprint +import pkg_resources import sys from . import hookspecs -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources -if sys.version_info >= (3, 10): - import importlib.metadata as importlib_metadata -else: - import importlib_metadata - - DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -23,45 +13,16 @@ DEFAULT_PLUGINS = ( "datasette.sql_functions", "datasette.actor_auth_cookie", "datasette.default_permissions", - "datasette.default_permissions.tokens", - "datasette.default_actions", - "datasette.default_column_types", "datasette.default_magic_parameters", "datasette.blob_renderer", - "datasette.default_debug_menu", - "datasette.default_jump_items", - "datasette.default_database_actions", + "datasette.default_menu_links", "datasette.handle_exception", "datasette.forbidden", - "datasette.events", ) pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) -DATASETTE_TRACE_PLUGINS = os.environ.get("DATASETTE_TRACE_PLUGINS", None) - - -def before(hook_name, hook_impls, kwargs): - print(file=sys.stderr) - print(f"{hook_name}:", file=sys.stderr) - pprint(kwargs, width=40, indent=4, stream=sys.stderr) - print("Hook implementations:", file=sys.stderr) - pprint(hook_impls, width=40, indent=4, stream=sys.stderr) - - -def after(outcome, hook_name, hook_impls, kwargs): - results = outcome.get_result() - if not isinstance(results, list): - results = [results] - print("Results:", file=sys.stderr) - pprint(results, width=40, indent=4, stream=sys.stderr) - - -if DATASETTE_TRACE_PLUGINS: - pm.add_hookcall_monitoring(before, after) - - DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: @@ -74,15 +35,15 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = importlib_metadata.distribution(package_name) - entry_points = distribution.entry_points - for entry_point in entry_points: - if entry_point.group == "datasette": + distribution = pkg_resources.get_distribution(package_name) + entry_map = distribution.get_entry_map() + if "datasette" in entry_map: + for plugin_name, entry_point in entry_map["datasette"].items(): mod = entry_point.load() pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except importlib_metadata.PackageNotFoundError: + except pkg_resources.DistributionNotFound: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -98,24 +59,21 @@ def get_plugins(): for plugin in pm.get_plugins(): static_path = None templates_path = None - plugin_name = ( - plugin.__name__ - if hasattr(plugin, "__name__") - else plugin.__class__.__name__ - ) - if plugin_name not in DEFAULT_PLUGINS: + if plugin.__name__ not in DEFAULT_PLUGINS: try: - if (importlib_resources.files(plugin_name) / "static").is_dir(): - static_path = str(importlib_resources.files(plugin_name) / "static") - if (importlib_resources.files(plugin_name) / "templates").is_dir(): - templates_path = str( - importlib_resources.files(plugin_name) / "templates" + if pkg_resources.resource_isdir(plugin.__name__, "static"): + static_path = pkg_resources.resource_filename( + plugin.__name__, "static" ) - except (TypeError, ModuleNotFoundError): - # Caused by --plugins_dir= plugins + if pkg_resources.resource_isdir(plugin.__name__, "templates"): + templates_path = pkg_resources.resource_filename( + plugin.__name__, "templates" + ) + except (KeyError, ImportError): + # Caused by --plugins_dir= plugins - KeyError/ImportError thrown in Py3.5 pass plugin_info = { - "name": plugin_name, + "name": plugin.__name__, "static_path": static_path, "templates_path": templates_path, "hooks": [h.name for h in pm.get_hookcallers(plugin)], @@ -123,6 +81,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.name or distinfo.project_name + plugin_info["name"] = distinfo.project_name plugins.append(plugin_info) return plugins diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 63d22fe8..760ff0d1 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -3,7 +3,7 @@ import click import json import os import re -from subprocess import CalledProcessError, check_call, check_output +from subprocess import check_call, check_output from .common import ( add_common_publish_arguments_and_options, @@ -23,9 +23,7 @@ def publish_subcommand(publish): help="Application name to use when building", ) @click.option( - "--service", - default="", - help="Cloud Run service to deploy (or over-write)", + "--service", default="", help="Cloud Run service to deploy (or over-write)" ) @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") @click.option( @@ -57,32 +55,13 @@ def publish_subcommand(publish): @click.option( "--max-instances", type=int, - default=1, - show_default=True, - help="Maximum Cloud Run instances (use 0 to remove the limit)", + help="Maximum Cloud Run instances", ) @click.option( "--min-instances", type=int, help="Minimum Cloud Run instances", ) - @click.option( - "--artifact-repository", - default="datasette", - show_default=True, - help="Artifact Registry repository to store the image", - ) - @click.option( - "--artifact-region", - default="us", - show_default=True, - help="Artifact Registry location (region or multi-region)", - ) - @click.option( - "--artifact-project", - default=None, - help="Project ID for Artifact Registry (defaults to the active project)", - ) def cloudrun( files, metadata, @@ -112,9 +91,6 @@ def publish_subcommand(publish): apt_get_extras, max_instances, min_instances, - artifact_repository, - artifact_region, - artifact_project, ): "Publish databases to Datasette running on Cloud Run" fail_if_publish_binary_not_installed( @@ -124,21 +100,6 @@ def publish_subcommand(publish): "gcloud config get-value project", shell=True, universal_newlines=True ).strip() - artifact_project = artifact_project or project - - # Ensure Artifact Registry exists for the target image - _ensure_artifact_registry( - artifact_project=artifact_project, - artifact_region=artifact_region, - artifact_repository=artifact_repository, - ) - - artifact_host = ( - artifact_region - if artifact_region.endswith("-docker.pkg.dev") - else f"{artifact_region}-docker.pkg.dev" - ) - if not service: # Show the user their current services, then prompt for one click.echo("Please provide a service name for this deployment\n") @@ -156,11 +117,6 @@ def publish_subcommand(publish): click.echo("") service = click.prompt("Service name", type=str) - image_id = ( - f"{artifact_host}/{artifact_project}/" - f"{artifact_repository}/datasette-{service}" - ) - extra_metadata = { "title": title, "license": license, @@ -217,6 +173,7 @@ def publish_subcommand(publish): print(fp.read()) print("\n====================\n") + image_id = f"gcr.io/{project}/datasette-{service}" check_call( "gcloud builds submit --tag {}{}".format( image_id, " --timeout {}".format(timeout) if timeout else "" @@ -230,7 +187,7 @@ def publish_subcommand(publish): ("--max-instances", max_instances), ("--min-instances", min_instances), ): - if value is not None: + if value: extra_deploy_options.append("{} {}".format(option, value)) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( @@ -242,52 +199,6 @@ def publish_subcommand(publish): ) -def _ensure_artifact_registry(artifact_project, artifact_region, artifact_repository): - """Ensure Artifact Registry API is enabled and the repository exists.""" - - enable_cmd = ( - "gcloud services enable artifactregistry.googleapis.com " - f"--project {artifact_project} --quiet" - ) - try: - check_call(enable_cmd, shell=True) - except CalledProcessError as exc: - raise click.ClickException( - "Failed to enable artifactregistry.googleapis.com. " - "Please ensure you have permissions to manage services." - ) from exc - - describe_cmd = ( - "gcloud artifacts repositories describe {repo} --project {project} " - "--location {location} --quiet" - ).format( - repo=artifact_repository, - project=artifact_project, - location=artifact_region, - ) - try: - check_call(describe_cmd, shell=True) - return - except CalledProcessError: - create_cmd = ( - "gcloud artifacts repositories create {repo} --repository-format=docker " - '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' - ).format( - repo=artifact_repository, - location=artifact_region, - project=artifact_project, - ) - try: - check_call(create_cmd, shell=True) - click.echo(f"Created Artifact Registry repository '{artifact_repository}'") - except CalledProcessError as exc: - raise click.ClickException( - "Failed to create Artifact Registry repository. " - "Use --artifact-repository/--artifact-region to point to an existing repo " - "or create one manually." - ) from exc - - def get_existing_services(): services = json.loads( check_output( @@ -303,7 +214,6 @@ def get_existing_services(): "url": service["status"]["address"]["url"], } for service in services - if "url" in service["status"] ] diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..224031a7 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -20,7 +20,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): if column in json_cols: try: value = json.loads(value) - except (TypeError, ValueError): + except (TypeError, ValueError) as e: pass new_row.append(value) new_rows.append(new_row) @@ -56,6 +56,7 @@ def json_renderer(request, args, data, error, truncated=None): if truncated is not None: data["truncated"] = truncated + if shape == "arrayfirst": if not data["rows"]: data = [] @@ -67,7 +68,7 @@ def json_renderer(request, args, data, error, truncated=None): elif shape in ("objects", "object", "array"): columns = data.get("columns") rows = data.get("rows") - if rows and columns and not isinstance(rows[0], dict): + if rows and columns: data["rows"] = [dict(zip(columns, row)) for row in rows] if shape == "object": shape_error = None diff --git a/datasette/resources.py b/datasette/resources.py deleted file mode 100644 index ee2e6d98..00000000 --- a/datasette/resources.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Core resource types for Datasette's permission system.""" - -from datasette.permissions import Resource - - -class DatabaseResource(Resource): - """A database in Datasette.""" - - name = "database" - parent_class = None # Top of the resource hierarchy - - def __init__(self, database: str): - super().__init__(parent=database, child=None) - - @classmethod - async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT database_name AS parent, NULL AS child - FROM catalog_databases - """ - - -class TableResource(Resource): - """A table in a database.""" - - name = "table" - parent_class = DatabaseResource - - def __init__(self, database: str, table: str): - super().__init__(parent=database, child=table) - - @classmethod - async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT database_name AS parent, table_name AS child - FROM catalog_tables - UNION ALL - SELECT database_name AS parent, view_name AS child - FROM catalog_views - """ - - -class QueryResource(Resource): - """A stored query in a database.""" - - name = "query" - parent_class = DatabaseResource - - def __init__(self, database: str, query: str): - super().__init__(parent=database, child=query) - - @classmethod - async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT q.database_name AS parent, q.name AS child - FROM queries q - JOIN catalog_databases cd ON cd.database_name = q.database_name - """ diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..80dfc677 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -63,14 +63,6 @@ em { } /* end reset */ -/* Modal CSS variables (shared by web components via Shadow DOM) */ -:root { - --modal-backdrop-bg: rgba(0, 0, 0, 0.5); - --modal-backdrop-blur: blur(4px); - --modal-border-radius: 0.75rem; - --modal-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); - --modal-animation-duration: 0.2s; -} body { margin: 0; @@ -171,22 +163,28 @@ h6, } .page-header { + display: flex; + align-items: center; padding-left: 10px; border-left: 10px solid #666; margin-bottom: 0.75rem; margin-top: 1rem; } .page-header h1 { + display: inline; margin: 0; font-size: 2rem; padding-right: 0.2em; } - -.page-action-menu details > summary { +.page-header details { + display: inline-flex; +} +.page-header details > summary { list-style: none; + display: inline-flex; cursor: pointer; } -.page-action-menu details > summary::-webkit-details-marker { +.page-header details > summary::-webkit-details-marker { display: none; } @@ -230,6 +228,12 @@ button.button-as-link:focus { color: #67C98D; } +a img { + display: block; + max-width: 100%; + border: 0; +} + code, pre { font-family: monospace; @@ -267,28 +271,24 @@ a.not-underlined { /* Page Furniture ========================================================= */ /* Header */ -header.hd, -footer.ft { +header, +footer { padding: 0.6rem 1rem 0.5rem 1rem; background-color: #276890; - background: linear-gradient(180deg, rgba(96,144,173,1) 0%, rgba(39,104,144,1) 50%); color: rgba(255,255,244,0.9); overflow: hidden; box-sizing: border-box; min-height: 2.6rem; } -footer.ft { - margin-top: 1rem; -} -header.hd p, -footer.ft p { +header p, +footer p { margin: 0; padding: 0; } -header.hd .crumbs { +header .crumbs { float: left; } -header.hd .actor { +header .actor { float: right; text-align: right; padding-left: 1rem; @@ -297,32 +297,32 @@ header.hd .actor { top: -3px; } -footer.ft a:link, -footer.ft a:visited, -footer.ft a:hover, -footer.ft a:focus, -footer.ft a:active, -footer.ft button.button-as-link { +footer a:link, +footer a:visited, +footer a:hover, +footer a:focus, +footer a:active, +footer button.button-as-link { color: rgba(255,255,244,0.8); } -header.hd a:link, -header.hd a:visited, -header.hd a:hover, -header.hd a:focus, -header.hd a:active, -header.hd button.button-as-link { +header a:link, +header a:visited, +header a:hover, +header a:focus, +header a:active, +header button.button-as-link { color: rgba(255,255,244,0.8); text-decoration: none; } -footer.ft a:hover, -footer.ft a:focus, -footer.ft a:active, -footer.ft .button-as-link:hover, -footer.ft .button-as-link:focus, -header.hd a:hover, -header.hd a:focus, -header.hd a:active, +footer a:hover, +footer a:focus, +footer a:active, +footer.button-as-link:hover, +footer.button-as-link:focus, +header a:hover, +header a:focus, +header a:active, button.button-as-link:hover, button.button-as-link:focus { color: rgba(255,255,244,1); @@ -334,6 +334,11 @@ section.content { margin: 0 1rem; } +/* Footer */ +footer { + margin-top: 1rem; +} + /* Navigation menu */ details.nav-menu > summary { list-style: none; @@ -347,85 +352,25 @@ details.nav-menu > summary::-webkit-details-marker { } details .nav-menu-inner { position: absolute; - top: 2.6rem; + top: 2rem; right: 10px; width: 180px; background-color: #276890; + padding: 1rem; z-index: 1000; - padding: 0; -} -.nav-menu-inner li, -form.nav-menu-logout { - padding: 0.3rem 0.5rem; - border-top: 1px solid #ffffff69; } .nav-menu-inner a { display: block; } -.nav-menu-inner button.button-as-link { - display: block; - width: 100%; - text-align: left; - font: inherit; -} -.nav-menu-inner .keyboard-shortcut { - float: right; - box-sizing: border-box; - min-width: 1.4em; - margin-left: 0.75rem; - padding: 0 0.35em; - border: 1px solid rgba(255,255,244,0.6); - border-radius: 3px; - background: rgba(255,255,244,0.12); - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.85em; - line-height: 1.35; - text-align: center; - text-decoration: none; -} -@media (max-width: 640px) { - .nav-menu-inner .keyboard-shortcut { - display: none; - } -} /* Table/database actions menu */ -.page-action-menu { +.page-header { position: relative; - margin-bottom: 0.5em; -} -.actions-menu-links { - display: inline; } .actions-menu-links .dropdown-menu { position: absolute; top: calc(100% + 10px); - left: 0; - z-index: 10000; -} -.page-action-menu .icon-text { - display: inline-flex; - align-items: center; - border-radius: .25rem; - padding: 5px 12px 3px 7px; - color: #fff; - font-weight: 400; - font-size: 0.8em; - background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); - border-color: #007bff; -} -.page-action-menu .icon-text span { - /* Nudge text up a bit */ - position: relative; - top: -2px; -} -.page-action-menu .icon-text:hover { - cursor: pointer; -} -.page-action-menu .icon { - width: 18px; - height: 18px; - margin-right: 4px; + left: -10px; } /* Components ============================================================== */ @@ -478,30 +423,36 @@ h2 em { .table-wrapper { overflow-x: auto; } -table.rows-and-columns { +table { border-collapse: collapse; } -table.rows-and-columns td { +td { border-top: 1px solid #aaa; border-right: 1px solid #eee; padding: 4px; vertical-align: top; white-space: pre-wrap; } -table.rows-and-columns td.type-pk { +td.type-pk { font-weight: bold; } -table.rows-and-columns td em { +td em { font-style: normal; font-size: 0.8em; color: #aaa; } -table.rows-and-columns th { +th { padding-right: 1em; } -table.rows-and-columns a:link { +table a:link { text-decoration: none; } +.rows-and-columns td:before { + display: block; + color: black; + margin-left: -10%; + font-size: 0.8em; +} .rows-and-columns td ol, .rows-and-columns td ul { list-style: initial; @@ -519,8 +470,10 @@ a.blob-download { margin-bottom: 0; } + /* Forms =================================================================== */ + form.sql textarea { border: 1px solid #ccc; width: 70%; @@ -532,6 +485,10 @@ form.sql textarea { form.sql label { width: 15%; } +form label { + font-weight: bold; + display: inline-block; +} .advanced-export input[type=submit] { font-size: 0.6em; margin-left: 1em; @@ -542,17 +499,8 @@ label.sort_by_desc { pre#sql-query { margin-bottom: 1em; } - -.core label, -label.core { - font-weight: bold; - display: inline-block; -} - -.core input[type=text], -input.core[type=text], -.core input[type=search], -input.core[type=search] { +form input[type=text], +form input[type=search] { border: 1px solid #ccc; border-radius: 3px; width: 60%; @@ -561,27 +509,19 @@ input.core[type=search] { font-size: 1em; font-family: Helvetica, sans-serif; } -.core input[type=search], -input.core[type=search] { - /* Stop Webkit from styling search boxes in an inconsistent way */ - /* https://css-tricks.com/webkit-html5-search-inputs/ comments */ +/* Stop Webkit from styling search boxes in an inconsistent way */ +/* https://css-tricks.com/webkit-html5-search-inputs/ comments */ +input[type=search] { -webkit-appearance: textfield; } -.core input[type="search"]::-webkit-search-decoration, -input.core[type="search"]::-webkit-search-decoration, -.core input[type="search"]::-webkit-search-cancel-button, -input.core[type="search"]::-webkit-search-cancel-button, -.core input[type="search"]::-webkit-search-results-button, -input.core[type="search"]::-webkit-search-results-button, -.core input[type="search"]::-webkit-search-results-decoration, -input.core[type="search"]::-webkit-search-results-decoration { +input[type="search"]::-webkit-search-decoration, +input[type="search"]::-webkit-search-cancel-button, +input[type="search"]::-webkit-search-results-button, +input[type="search"]::-webkit-search-results-decoration { display: none; } -.core input[type=submit], -.core button[type=button], -input.core[type=submit], -button.core[type=button] { +form input[type=submit], form button[type=button] { font-weight: 400; cursor: pointer; text-align: center; @@ -594,16 +534,14 @@ button.core[type=button] { border-radius: .25rem; } -.core input[type=submit], -input.core[type=submit] { +form input[type=submit] { color: #fff; - background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); + background-color: #007bff; border-color: #007bff; -webkit-appearance: button; } -.core button[type=button], -button.core[type=button] { +form button[type=button] { color: #007bff; background-color: #fff; border-color: #007bff; @@ -681,14 +619,10 @@ button.core[type=button] { border-radius: 3px; -webkit-appearance: none; padding: 9px 4px; - font-size: 16px; + font-size: 1em; font-family: Helvetica, sans-serif; } -#_search { - font-size: 16px; -} - @@ -768,474 +702,6 @@ p.zero-results { .select-wrapper.small-screen-only { display: none; } - -@keyframes datasette-modal-slide-in { - from { - opacity: 0; - transform: translateY(-20px) scale(0.95); - } - to { - opacity: 1; - transform: translateY(0) scale(1); - } -} - -@keyframes datasette-modal-fade-in { - from { opacity: 0; } - to { opacity: 1; } -} - -dialog.mobile-column-actions-dialog { - --ink: #0f0f0f; - --paper: #f5f3ef; - --muted: #6b6b6b; - --rule: #e2dfd8; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(420px, calc(100vw - 32px)); - max-width: 95vw; - max-height: min(640px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.mobile-column-actions-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.mobile-column-actions-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.mobile-column-actions-dialog .modal-header { - padding: 20px 24px 16px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.mobile-column-actions-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.mobile-column-actions-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -.mobile-column-actions-dialog .list-wrap { - flex: 1 1 auto; - min-height: 0; - overflow-y: auto; - overflow-x: hidden; - position: relative; - overscroll-behavior: contain; - -webkit-overflow-scrolling: touch; -} - -.mobile-column-actions-dialog .list-wrap::before, -.mobile-column-actions-dialog .list-wrap::after { - content: ""; - position: sticky; - display: block; - left: 0; - right: 0; - height: 20px; - pointer-events: none; - z-index: 5; -} - -.mobile-column-actions-dialog .list-wrap::before { - top: 0; - background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent); -} - -.mobile-column-actions-dialog .list-wrap::after { - bottom: 0; - background: linear-gradient(to top, rgba(255,255,255,0.9), transparent); - margin-top: -20px; -} - -.mobile-column-top-actions { - padding: 10px 24px 0; -} - -.mobile-column-top-action { - display: inline-block; - text-decoration: none; -} - -.mobile-column-section { - border-bottom: 1px solid var(--rule); -} - -.mobile-column-actions-dialog .col-header { - width: 100%; - padding: 12px 24px; - font: inherit; - font-weight: 600; - border: 0; - background: none; - cursor: pointer; - display: flex; - justify-content: space-between; - align-items: center; - text-align: left; -} - -.mobile-column-header-text { - display: flex; - flex-direction: column; - gap: 0.15rem; -} - -.mobile-column-name { - color: var(--ink); -} - -.mobile-column-meta { - color: var(--muted); - font-size: 0.78em; - font-family: ui-monospace, monospace; - font-weight: normal; -} - -.mobile-column-chevron { - color: var(--muted); - transition: transform 0.2s ease-out; -} - -.mobile-column-actions-dialog .col-header[aria-expanded="true"] .mobile-column-chevron { - transform: rotate(180deg); -} - -.mobile-column-actions-dialog .col-actions[hidden] { - display: none; -} - -.mobile-column-actions-dialog .col-actions ul, -.mobile-column-actions-dialog .col-actions li { - margin: 0; - padding: 0; - list-style-type: none; -} - -.mobile-column-actions-dialog .col-actions a, -.mobile-column-actions-dialog .col-actions button { - display: block; - width: 100%; - padding: 10px 24px 10px 40px; - color: var(--ink); - text-align: left; - font: inherit; - text-decoration: none; - background: none; - border: 0; - border-top: 1px solid #f5f5f5; - cursor: pointer; -} - -.mobile-column-actions-dialog .col-actions a:hover, -.mobile-column-actions-dialog .col-actions button:hover { - background: var(--paper); -} - -.mobile-column-actions-dialog .col-actions a:active, -.mobile-column-actions-dialog .col-actions button:active { - background: #eee; -} - -.mobile-column-description, -.mobile-column-no-actions { - margin: 0; - padding: 0 24px 12px 24px; - color: var(--muted); - font-size: 0.85em; -} - -.mobile-column-actions-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.mobile-column-actions-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.mobile-column-actions-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.mobile-column-actions-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.mobile-column-actions-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -dialog.set-column-type-dialog { - --ink: #0f0f0f; - --paper: #f5f3ef; - --muted: #6b6b6b; - --rule: #e2dfd8; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; - max-height: min(720px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.set-column-type-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.set-column-type-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.set-column-type-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.set-column-type-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.set-column-type-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -.set-column-type-status, -.set-column-type-empty, -.set-column-type-error { - margin: 0; - padding: 12px 24px 0; -} - -.set-column-type-status, -.set-column-type-empty { - color: var(--muted); - font-size: 0.9rem; -} - -.set-column-type-error { - color: #b91c1c; - font-size: 0.9rem; -} - -.set-column-type-options { - padding: 16px 24px 24px; - overflow-y: auto; - display: grid; - gap: 12px; -} - -.set-column-type-option { - display: grid; - grid-template-columns: auto 1fr; - gap: 12px; - align-items: start; - padding: 14px 16px; - border: 1px solid var(--rule); - border-radius: 8px; - background: #fcfbf9; - cursor: pointer; -} - -.set-column-type-option:focus-within { - border-color: var(--accent); - box-shadow: 0 0 0 3px rgba(26, 86, 219, 0.12); -} - -.set-column-type-option input { - margin-top: 3px; -} - -.set-column-type-option-content { - display: grid; - gap: 4px; -} - -.set-column-type-option-name { - font-family: ui-monospace, monospace; - font-size: 0.95rem; - color: var(--ink); -} - -.set-column-type-option-description { - color: var(--muted); - font-size: 0.9rem; -} - -.set-column-type-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.set-column-type-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.set-column-type-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.set-column-type-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.set-column-type-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.set-column-type-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.set-column-type-dialog .btn-primary:hover { - background: #1949b8; -} - -.set-column-type-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - -@media (max-width: 640px) { - dialog.mobile-column-actions-dialog { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; - } - - .mobile-column-actions-dialog .modal-header { - padding: 16px 18px 14px; - } - - .mobile-column-top-actions { - padding-left: 18px; - padding-right: 18px; - } - - .mobile-column-actions-dialog .col-header { - padding-left: 18px; - padding-right: 18px; - } - - .mobile-column-actions-dialog .col-actions a, - .mobile-column-actions-dialog .col-actions button { - padding-left: 34px; - padding-right: 18px; - } - - .mobile-column-description, - .mobile-column-no-actions { - padding-left: 18px; - padding-right: 18px; - } - - dialog.set-column-type-dialog { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; - } - - .set-column-type-dialog .modal-header, - .set-column-type-status, - .set-column-type-empty, - .set-column-type-error, - .set-column-type-options { - padding-left: 18px; - padding-right: 18px; - } -} - @media only screen and (max-width: 576px) { .small-screen-only { @@ -1265,7 +731,7 @@ dialog.set-column-type-dialog::backdrop { left: -9999px; } - table.rows-and-columns tr { + .rows-and-columns tr { border: 1px solid #ccc; margin-bottom: 1em; border-radius: 10px; @@ -1273,7 +739,7 @@ dialog.set-column-type-dialog::backdrop { padding: 0.2rem; } - table.rows-and-columns td { + .rows-and-columns td { /* Behave like a "row" */ border: none; border-bottom: 1px solid #eee; @@ -1281,7 +747,7 @@ dialog.set-column-type-dialog::backdrop { padding-left: 10%; } - table.rows-and-columns td:before { + .rows-and-columns td:before { display: block; color: black; margin-left: -10%; @@ -1297,43 +763,6 @@ dialog.set-column-type-dialog::backdrop { .filters input.filter-value { width: 140px; } - button.choose-columns-mobile, - button.column-actions-mobile { - display: inline-flex; - align-items: center; - justify-content: center; - padding: 0.5rem 1rem; - margin-bottom: 1em; - font-size: 0.9rem; - line-height: 1.2; - font-family: inherit; - background: white; - border: 1px solid #ccc; - border-radius: 5px; - cursor: pointer; - vertical-align: top; - box-sizing: border-box; - min-height: 2.5rem; - } - - button.column-actions-mobile { - gap: 0.55rem; - } - - button.column-actions-mobile svg { - display: block; - width: 16px; - height: 16px; - flex-shrink: 0; - } - - button.column-actions-mobile span { - line-height: 1.2; - } - - button.choose-columns-mobile { - margin-right: 0.5rem; - } } svg.dropdown-menu-icon { @@ -1390,13 +819,6 @@ svg.dropdown-menu-icon { .dropdown-menu a:hover { background-color: #eee; } -.dropdown-menu .dropdown-description { - margin: 0; - color: #666; - font-size: 0.8em; - max-width: 80vw; - white-space: normal; -} .dropdown-menu .hook { display: block; position: absolute; @@ -1409,15 +831,11 @@ svg.dropdown-menu-icon { border-bottom: 5px solid #666; } -.stored-query-edit-sql { +.canned-query-edit-sql { padding-left: 0.5em; position: relative; top: 1px; } -.save-query { - display: inline-block; - margin-left: 0.45em; -} .blob-download { display: block; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js deleted file mode 100644 index 133e7cb0..00000000 --- a/datasette/static/column-chooser.js +++ /dev/null @@ -1,699 +0,0 @@ -class ColumnChooser extends HTMLElement { - constructor() { - super(); - this.attachShadow({ mode: "open" }); - - // State - this._items = []; - this._checked = new Set(); - this._savedItems = null; - this._savedChecked = null; - this._onApply = null; - - // Drag state - this._ghost = null; - this._dragSrcIdx = null; - this._dropTargetIdx = null; - this._dropPosition = null; - this._ghostOffX = 0; - this._ghostOffY = 0; - this._autoScrollRAF = null; - this._lastPointerY = 0; - this._lastPointerX = 0; - this._SCROLL_ZONE = 72; - this._SCROLL_SPEED = 0.4; - - // Bound handlers - this._onMove = this._onMove.bind(this); - this._onUp = this._onUp.bind(this); - - this.shadowRoot.innerHTML = ` - - - -
- Choose columns - -
-
- - -
-
-
-
-
    -
    -
    - - - -
    -     - `; - - // DOM refs - this._dialog = this.shadowRoot.querySelector("dialog"); - this._listWrap = this.shadowRoot.getElementById("listWrap"); - this._dragList = this.shadowRoot.getElementById("dragList"); - this._pulseTop = this.shadowRoot.getElementById("pulseTop"); - this._pulseBot = this.shadowRoot.getElementById("pulseBot"); - this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); - this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); - this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); - this._applyBtn = this.shadowRoot.getElementById("applyBtn"); - this._countEl = this.shadowRoot.getElementById("selectedCount"); - this._footerEl = this.shadowRoot.getElementById("footerInfo"); - - // Event listeners - this._selectAllBtn.addEventListener("click", () => this._selectAll()); - this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); - this._cancelBtn.addEventListener("click", () => this._close()); - this._applyBtn.addEventListener("click", () => this._apply()); - this._dialog.addEventListener("click", (e) => { - if (e.target === this._dialog) this._close(); - }); - this._dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this._close(); - }); - } - - /** - * Open the column chooser dialog. - * @param {Object} opts - * @param {string[]} opts.columns - All available column names, in display order. - * @param {string[]} opts.selected - Column names that should be pre-checked. - * @param {function(string[]): void} opts.onApply - Called with the selected columns in order when Apply is clicked. - */ - open({ columns, selected = [], onApply }) { - this._items = [...columns]; - this._checked = new Set(selected); - this._onApply = onApply || null; - - // Save state for cancel/restore - this._savedItems = [...this._items]; - this._savedChecked = new Set(this._checked); - - this._render(); - this._dialog.showModal(); - } - - // ── Internal methods ── - - _close() { - this._items = this._savedItems ? [...this._savedItems] : this._items; - this._checked = this._savedChecked - ? new Set(this._savedChecked) - : this._checked; - this._dialog.close(); - } - - _selectAll() { - this._items.forEach((col) => this._checked.add(col)); - this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { - cb.checked = true; - }); - this._updateCounts(); - } - - _deselectAll() { - this._checked.clear(); - this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { - cb.checked = false; - }); - this._updateCounts(); - } - - _apply() { - const selected = this._items.filter((col) => this._checked.has(col)); - this._dialog.close(); - if (this._onApply) { - this._onApply(selected); - } - } - - _render() { - this._dragList.innerHTML = ""; - this._items.forEach((col, i) => { - const li = document.createElement("li"); - li.className = "drag-item"; - li.dataset.idx = i; - li.innerHTML = ` - - - - - - - - - - - -
    - `; - - li.querySelector("input").addEventListener("change", (e) => { - e.target.checked ? this._checked.add(col) : this._checked.delete(col); - this._updateCounts(); - }); - - li.querySelector(".drag-handle").addEventListener("pointerdown", (e) => - this._startDrag(e, i), - ); - this._dragList.appendChild(li); - }); - - this._updateCounts(); - } - - _updateCounts() { - const n = this._checked.size; - this._countEl.textContent = `${n} of ${this._items.length} selected`; - this._footerEl.textContent = `${this._items.length} columns`; - } - - // ── Drag engine ── - - _startDrag(e, idx) { - e.preventDefault(); - this._dragSrcIdx = idx; - - const srcEl = this._dragList.children[idx]; - const rect = srcEl.getBoundingClientRect(); - - this._ghostOffX = e.clientX - rect.left; - this._ghostOffY = e.clientY - rect.top; - - // Build ghost inside shadow DOM - this._ghost = document.createElement("div"); - this._ghost.className = "drag-ghost"; - this._ghost.style.width = rect.width + "px"; - this._ghost.style.height = rect.height + "px"; - this._ghost.innerHTML = srcEl.innerHTML; - this._ghost.querySelector(".drop-indicator")?.remove(); - const h = this._ghost.querySelector(".drag-handle"); - if (h) h.style.color = "var(--accent)"; - this.shadowRoot.appendChild(this._ghost); - - srcEl.classList.add("is-dragging"); - this._positionGhost(e.clientX, e.clientY); - - document.addEventListener("pointermove", this._onMove); - document.addEventListener("pointerup", this._onUp); - document.addEventListener("pointercancel", this._onUp); - } - - _positionGhost(cx, cy) { - this._ghost.style.left = cx - this._ghostOffX + "px"; - this._ghost.style.top = cy - this._ghostOffY + "px"; - } - - _onMove(e) { - this._lastPointerX = e.clientX; - this._lastPointerY = e.clientY; - this._positionGhost(e.clientX, e.clientY); - this._updateDropTarget(e.clientY); - this._updateAutoScroll(e.clientY); - } - - _onUp() { - document.removeEventListener("pointermove", this._onMove); - document.removeEventListener("pointerup", this._onUp); - document.removeEventListener("pointercancel", this._onUp); - - this._stopAutoScroll(); - - const noMove = - this._dropTargetIdx === null || this._dropTargetIdx === this._dragSrcIdx; - this._clearDropIndicators(); - - let dest = null; - if (!noMove) { - const moved = this._items.splice(this._dragSrcIdx, 1)[0]; - dest = this._dropTargetIdx; - if (this._dropPosition === "after") dest++; - if (dest > this._dragSrcIdx) dest--; - this._items.splice(dest, 0, moved); - } - - this._dragSrcIdx = null; - this._dropTargetIdx = null; - this._dropPosition = null; - - const g = this._ghost; - this._ghost = null; - - if (noMove) { - if (g) g.remove(); - this._render(); - return; - } - - this._render(); - - if (g && dest !== null) { - const landedEl = this._dragList.children[dest]; - if (landedEl) { - landedEl.style.opacity = "0"; - const r = landedEl.getBoundingClientRect(); - g.getBoundingClientRect(); - g.style.transition = - "left 0.15s cubic-bezier(0.22, 1, 0.36, 1), top 0.15s cubic-bezier(0.22, 1, 0.36, 1), box-shadow 0.15s, opacity 0.1s 0.1s"; - g.style.left = r.left + "px"; - g.style.top = r.top + "px"; - g.style.boxShadow = "0 1px 4px rgba(0,0,0,0.08)"; - g.style.opacity = "0"; - setTimeout(() => { - g.remove(); - if (landedEl) landedEl.style.opacity = ""; - }, 160); - } else { - g.remove(); - } - } else if (g) { - g.remove(); - } - } - - _updateDropTarget(clientY) { - this._clearDropIndicators(); - const listItems = [ - ...this._dragList.querySelectorAll(".drag-item:not(.is-dragging)"), - ]; - if (!listItems.length) return; - - let best = null, - bestDist = Infinity; - listItems.forEach((li) => { - const r = li.getBoundingClientRect(); - const mid = r.top + r.height / 2; - const dist = Math.abs(clientY - mid); - if (dist < bestDist) { - bestDist = dist; - best = li; - } - }); - - if (!best) return; - const r = best.getBoundingClientRect(); - const mid = r.top + r.height / 2; - const above = clientY < mid; - const indic = best.querySelector(".drop-indicator"); - - this._dropTargetIdx = parseInt(best.dataset.idx); - this._dropPosition = above ? "before" : "after"; - - if (indic) { - indic.className = "drop-indicator " + (above ? "top" : "bottom"); - } - } - - _clearDropIndicators() { - this._dragList.querySelectorAll(".drop-indicator").forEach((el) => { - el.className = "drop-indicator"; - }); - } - - _updateAutoScroll(clientY) { - const rect = this._listWrap.getBoundingClientRect(); - const relY = clientY - rect.top; - const distTop = relY; - const distBot = rect.height - relY; - - const inTop = distTop < this._SCROLL_ZONE && distTop >= 0; - const inBot = distBot < this._SCROLL_ZONE && distBot >= 0; - - this._pulseTop.classList.toggle("active", inTop); - this._pulseBot.classList.toggle("active", inBot); - - if ((inTop || inBot) && !this._autoScrollRAF) { - let lastTime = null; - const loop = (ts) => { - if (!this._ghost) { - this._stopAutoScroll(); - return; - } - if (lastTime !== null) { - const dt = ts - lastTime; - const rect2 = this._listWrap.getBoundingClientRect(); - const relY2 = this._lastPointerY - rect2.top; - const dTop = relY2; - const dBot = rect2.height - relY2; - - if (dTop < this._SCROLL_ZONE && dTop >= 0) { - const factor = 1 - dTop / this._SCROLL_ZONE; - this._listWrap.scrollTop -= this._SCROLL_SPEED * dt * factor * 2.5; - } else if (dBot < this._SCROLL_ZONE && dBot >= 0) { - const factor = 1 - dBot / this._SCROLL_ZONE; - this._listWrap.scrollTop += this._SCROLL_SPEED * dt * factor * 2.5; - } else { - this._stopAutoScroll(); - return; - } - this._updateDropTarget(this._lastPointerY); - } - lastTime = ts; - this._autoScrollRAF = requestAnimationFrame(loop); - }; - this._autoScrollRAF = requestAnimationFrame(loop); - } - - if (!inTop && !inBot) this._stopAutoScroll(); - } - - _stopAutoScroll() { - if (this._autoScrollRAF) { - cancelAnimationFrame(this._autoScrollRAF); - this._autoScrollRAF = null; - } - this._pulseTop.classList.remove("active"); - this._pulseBot.classList.remove("active"); - } -} - -customElements.define("column-chooser", ColumnChooser); diff --git a/datasette/static/datasette-manager.js b/datasette/static/datasette-manager.js deleted file mode 100644 index e75f7aae..00000000 --- a/datasette/static/datasette-manager.js +++ /dev/null @@ -1,222 +0,0 @@ -// Custom events for use with the native CustomEvent API -const DATASETTE_EVENTS = { - INIT: "datasette_init", // returns datasette manager instance in evt.detail -}; - -// Datasette "core" -> Methods/APIs that are foundational -// Plugins will have greater stability if they use the functional hooks- but if they do decide to hook into -// literal DOM selectors, they'll have an easier time using these addresses. -const DOM_SELECTORS = { - /** Should have one match */ - jsonExportLink: ".export-links a[href*=json]", - - /** Event listeners that go outside of the main table, e.g. existing scroll listener */ - tableWrapper: ".table-wrapper", - table: "table.rows-and-columns", - aboveTablePanel: ".above-table-panel", - - // These could have multiple matches - /** Used for selecting table headers. Use makeColumnActions if you want to add menu items. */ - tableHeaders: `table.rows-and-columns th`, - - /** Used to add "where" clauses to query using direct manipulation */ - filterRows: ".filter-row", - /** Used to show top available enum values for a column ("facets") */ - facetResults: ".facet-results [data-column]", -}; - -/** - * Monolith class for interacting with Datasette JS API - * Imported with DEFER, runs after main document parsed - * For now, manually synced with datasette/version.py - */ -const datasetteManager = { - VERSION: window.datasetteVersion, - - // TODO: Should order of registration matter more? - - // Should plugins be allowed to clobber others or is it last-in takes priority? - // Does pluginMetadata need to be serializable, or can we let it be stateful / have functions? - plugins: new Map(), - - registerPlugin: (name, pluginMetadata) => { - if (datasetteManager.plugins.has(name)) { - console.warn(`Warning -> plugin ${name} was redefined`); - } - datasetteManager.plugins.set(name, pluginMetadata); - - // If the plugin participates in the panel... update the panel. - if (pluginMetadata.makeAboveTablePanelConfigs) { - datasetteManager.renderAboveTablePanel(); - } - }, - - /** - * New DOM elements are created on each click, so the data is not stale. - * - * Items - * - must provide label (text) - * - might provide href (string) or an onclick ((evt) => void) - * - * columnMeta is metadata stored on the column header (TH) as a DOMStringMap - * - column: string - * - columnNotNull: boolean - * - columnType: sqlite datatype enum (text, number, etc) - * - isPk: boolean - */ - makeColumnActions: (columnMeta) => { - let columnActions = []; - - // Accept function that returns list of columnActions with keys - // Required: label (text) - // Optional: onClick or href - datasetteManager.plugins.forEach((plugin) => { - if (plugin.makeColumnActions) { - // Plugins can provide multiple columnActions if they want - // If multiple try to create entry with same label, the last one deletes the others - columnActions.push(...plugin.makeColumnActions(columnMeta)); - } - }); - - // TODO: Validate columnAction configs and give informative error message if missing keys. - return columnActions; - }, - - makeJumpSections: (context) => { - let jumpSections = []; - - datasetteManager.plugins.forEach((plugin) => { - if (plugin.makeJumpSections) { - const sections = plugin.makeJumpSections(context) || []; - jumpSections.push(...sections); - } - }); - - return jumpSections; - }, - - /** - * In MVP, each plugin can only have 1 instance. - * In future, panels could be repeated. We omit that for now since so many plugins depend on - * shared URL state, so having multiple instances of plugin at same time is problematic. - * Currently, we never destroy any panels, we just hide them. - * - * TODO: nicer panel css, show panel selection state. - * TODO: does this hook need to take any arguments? - */ - renderAboveTablePanel: () => { - const aboveTablePanel = document.querySelector( - DOM_SELECTORS.aboveTablePanel, - ); - - if (!aboveTablePanel) { - console.warn( - "This page does not have a table, the renderAboveTablePanel cannot be used.", - ); - return; - } - - let aboveTablePanelWrapper = aboveTablePanel.querySelector(".panels"); - - // First render: create wrappers. Otherwise, reuse previous. - if (!aboveTablePanelWrapper) { - aboveTablePanelWrapper = document.createElement("div"); - aboveTablePanelWrapper.classList.add("tab-contents"); - const panelNav = document.createElement("div"); - panelNav.classList.add("tab-controls"); - - // Temporary: css for minimal amount of breathing room. - panelNav.style.display = "flex"; - panelNav.style.gap = "8px"; - panelNav.style.marginTop = "4px"; - panelNav.style.marginBottom = "20px"; - - aboveTablePanel.appendChild(panelNav); - aboveTablePanel.appendChild(aboveTablePanelWrapper); - } - - datasetteManager.plugins.forEach((plugin, pluginName) => { - const { makeAboveTablePanelConfigs } = plugin; - - if (makeAboveTablePanelConfigs) { - const controls = aboveTablePanel.querySelector(".tab-controls"); - const contents = aboveTablePanel.querySelector(".tab-contents"); - - // Each plugin can make multiple panels - const configs = makeAboveTablePanelConfigs(); - - configs.forEach((config, i) => { - const nodeContentId = `${pluginName}_${config.id}_panel-content`; - - // quit if we've already registered this plugin - // TODO: look into whether plugins should be allowed to ask - // parent to re-render, or if they should manage that internally. - if (document.getElementById(nodeContentId)) { - return; - } - - // Add tab control button - const pluginControl = document.createElement("button"); - pluginControl.textContent = config.label; - pluginControl.onclick = () => { - contents.childNodes.forEach((node) => { - if (node.id === nodeContentId) { - node.style.display = "block"; - } else { - node.style.display = "none"; - } - }); - }; - controls.appendChild(pluginControl); - - // Add plugin content area - const pluginNode = document.createElement("div"); - pluginNode.id = nodeContentId; - config.render(pluginNode); - pluginNode.style.display = "none"; // Default to hidden unless you're ifrst - - contents.appendChild(pluginNode); - }); - - // Let first node be selected by default - if (contents.childNodes.length) { - contents.childNodes[0].style.display = "block"; - } - } - }); - }, - - /** Selectors for document (DOM) elements. Store identifier instead of immediate references in case they haven't loaded when Manager starts. */ - selectors: DOM_SELECTORS, - - // Future API ideas - // Fetch page's data in array, and cache so plugins could reuse it - // Provide knowledge of what datasette JS or server-side via traditional console autocomplete - // State helpers: URL params https://github.com/simonw/datasette/issues/1144 and localstorage - // UI Hooks: command + k, tab manager hook - // Should we notify plugins that have dependencies - // when all dependencies were fulfilled? (leaflet, codemirror, etc) - // https://github.com/simonw/datasette-leaflet -> this way - // multiple plugins can all request the same copy of leaflet. -}; - -const initializeDatasette = () => { - // Hide the global behind __ prefix. Ideally they should be listening for the - // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window. - - window.__DATASETTE__ = datasetteManager; - - const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, { - detail: datasetteManager, - }); - - document.dispatchEvent(initDatasetteEvent); -}; - -/** - * Main function - * Fires AFTER the document has been parsed - */ -document.addEventListener("DOMContentLoaded", function () { - initializeDatasette(); -}); diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js index 0e6e2c29..d83b8186 100644 --- a/datasette/static/json-format-highlight-1.0.1.js +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -7,8 +7,8 @@ MIT Licensed typeof exports === "object" && typeof module !== "undefined" ? (module.exports = factory()) : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); + ? define(factory) + : (global.jsonFormatHighlight = factory()); })(this, function () { "use strict"; @@ -42,13 +42,13 @@ MIT Licensed color = /true/.test(match) ? colors.trueColor : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; } return '' + match + ""; - }, + } ); } diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js deleted file mode 100644 index a386b1fc..00000000 --- a/datasette/static/mobile-column-actions.js +++ /dev/null @@ -1,318 +0,0 @@ -var MOBILE_COLUMN_BREAKPOINT = 576; -var MOBILE_COLUMN_DIALOG_ID = "mobile-column-actions-dialog"; -var MOBILE_COLUMN_DIALOG_TITLE_ID = "mobile-column-actions-title"; - -function mobileColumnHeaders(manager) { - return Array.from( - document.querySelectorAll(manager.selectors.tableHeaders), - ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1"); -} - -function mobileColumnMetaText(th) { - var parts = []; - if (th.dataset.columnType) { - parts.push(th.dataset.columnType); - } - if (th.dataset.isPk === "1") { - parts.push("pk"); - } - if (th.dataset.columnNotNull === "1") { - parts.push("not null"); - } - return parts.join(", "); -} - -function createMobileColumnActionNode(itemConfig, closeDialog) { - var actionNode; - if (itemConfig.href) { - actionNode = document.createElement("a"); - actionNode.href = itemConfig.href; - } else { - actionNode = document.createElement("button"); - actionNode.type = "button"; - } - actionNode.textContent = itemConfig.label; - - if (itemConfig.onClick) { - actionNode.addEventListener("click", function (ev) { - try { - itemConfig.onClick.call(actionNode, ev); - } finally { - closeDialog({ restoreFocus: false }); - } - }); - } - - return actionNode; -} - -function initMobileColumnActions(manager) { - var triggerButton = document.querySelector(".column-actions-mobile"); - if (!triggerButton) { - return; - } - - if ( - !window.URLSearchParams || - !window.HTMLDialogElement || - !manager.columnActions - ) { - triggerButton.style.display = "none"; - return; - } - - if (!mobileColumnHeaders(manager).length) { - triggerButton.style.display = "none"; - return; - } - - var dialog = document.createElement("dialog"); - dialog.className = "mobile-column-actions-dialog"; - dialog.id = MOBILE_COLUMN_DIALOG_ID; - dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); - dialog.innerHTML = ` -
    - Column actions - -
    -
    -
    - Tap a column to reveal actions. - -
    - `; - document.body.appendChild(dialog); - - triggerButton.setAttribute("aria-haspopup", "dialog"); - triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); - triggerButton.setAttribute("aria-expanded", "false"); - - var countEl = dialog.querySelector(".modal-meta"); - var listWrap = dialog.querySelector(".mobile-column-list"); - var doneButton = dialog.querySelector(".mobile-column-actions-done"); - var expandedSectionId = null; - var shouldRestoreFocus = true; - - function updateExpandedSection() { - Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { - var controlsId = button.getAttribute("aria-controls"); - var actionList = dialog.querySelector("#" + controlsId); - var isExpanded = controlsId === expandedSectionId; - button.setAttribute("aria-expanded", isExpanded ? "true" : "false"); - actionList.hidden = !isExpanded; - actionList.classList.toggle("expanded", isExpanded); - }); - } - - function scrollExpandedSectionIntoView(section) { - var sectionTop = section.offsetTop; - var sectionBottom = sectionTop + section.offsetHeight; - var visibleTop = listWrap.scrollTop; - var visibleBottom = visibleTop + listWrap.clientHeight; - var sectionHeight = section.offsetHeight; - - if (sectionTop < visibleTop) { - listWrap.scrollTop = sectionTop; - return; - } - - if (sectionBottom <= visibleBottom) { - return; - } - - if (sectionHeight <= listWrap.clientHeight) { - listWrap.scrollTop = sectionBottom - listWrap.clientHeight; - } else { - listWrap.scrollTop = sectionTop; - } - } - - function closeDialog(options) { - options = options || {}; - shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } - } - } - - function renderDialog() { - var headers = mobileColumnHeaders(manager); - if (!headers.length) { - closeDialog({ restoreFocus: false }); - triggerButton.style.display = "none"; - return false; - } - - if ( - !headers.some( - (_th, index) => `mobile-column-actions-${index}` === expandedSectionId, - ) - ) { - expandedSectionId = null; - } - - countEl.textContent = `${headers.length} column${ - headers.length === 1 ? "" : "s" - }`; - listWrap.innerHTML = ""; - - if (manager.columnActions.shouldShowShowAllColumns()) { - var topActions = document.createElement("div"); - topActions.className = "mobile-column-top-actions"; - - var showAllColumns = document.createElement("a"); - showAllColumns.className = "btn btn-ghost mobile-column-top-action"; - showAllColumns.href = manager.columnActions.showAllColumnsUrl(); - showAllColumns.textContent = "Show all columns"; - - topActions.appendChild(showAllColumns); - listWrap.appendChild(topActions); - } - - headers.forEach((th, index) => { - var sectionId = `mobile-column-actions-${index}`; - var actionState = manager.columnActions.buildColumnActionState(th, { - includeChooseColumns: false, - includeShowAllColumns: false, - }); - var section = document.createElement("section"); - section.className = "mobile-column-section"; - - var headerButton = document.createElement("button"); - headerButton.type = "button"; - headerButton.className = "col-header"; - headerButton.setAttribute("aria-controls", sectionId); - headerButton.setAttribute("aria-expanded", "false"); - - var headerText = document.createElement("span"); - headerText.className = "mobile-column-header-text"; - - var name = document.createElement("span"); - name.className = "mobile-column-name"; - name.textContent = th.dataset.column; - headerText.appendChild(name); - - var metaText = mobileColumnMetaText(th); - if (metaText) { - var meta = document.createElement("span"); - meta.className = "mobile-column-meta"; - meta.textContent = metaText; - headerText.appendChild(meta); - } - - var chevron = document.createElement("span"); - chevron.className = "mobile-column-chevron"; - chevron.setAttribute("aria-hidden", "true"); - chevron.textContent = "▾"; - - headerButton.appendChild(headerText); - headerButton.appendChild(chevron); - headerButton.addEventListener("click", function () { - expandedSectionId = expandedSectionId === sectionId ? null : sectionId; - updateExpandedSection(); - if (expandedSectionId === sectionId) { - scrollExpandedSectionIntoView(section); - } - }); - - var actionContainer = document.createElement("div"); - actionContainer.id = sectionId; - actionContainer.className = "col-actions"; - actionContainer.hidden = true; - - if (actionState.columnDescription) { - var description = document.createElement("p"); - description.className = "mobile-column-description"; - description.textContent = actionState.columnDescription; - actionContainer.appendChild(description); - } - - if (actionState.actionItems.length) { - var actionList = document.createElement("ul"); - actionState.actionItems.forEach((itemConfig) => { - var actionItem = document.createElement("li"); - actionItem.appendChild( - createMobileColumnActionNode(itemConfig, closeDialog), - ); - actionList.appendChild(actionItem); - }); - actionContainer.appendChild(actionList); - } else { - var noActions = document.createElement("p"); - noActions.className = "mobile-column-no-actions"; - noActions.textContent = "No actions available"; - actionContainer.appendChild(noActions); - } - - section.appendChild(headerButton); - section.appendChild(actionContainer); - listWrap.appendChild(section); - }); - - updateExpandedSection(); - return true; - } - - function openDialog() { - if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT) { - return; - } - if (!renderDialog()) { - return; - } - if (!dialog.open) { - dialog.showModal(); - } - triggerButton.setAttribute("aria-expanded", "true"); - var focusTarget = - dialog.querySelector(".mobile-column-top-action") || - dialog.querySelector(".col-header") || - doneButton; - focusTarget.focus(); - } - - triggerButton.addEventListener("click", function () { - if (dialog.open) { - closeDialog(); - } else { - openDialog(); - } - }); - - doneButton.addEventListener("click", function () { - closeDialog(); - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeDialog(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeDialog(); - }); - - dialog.addEventListener("close", function () { - triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } - }); - - window.addEventListener("resize", function () { - if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT && dialog.open) { - closeDialog({ restoreFocus: false }); - } - }); -} - -document.addEventListener("datasette_init", function (evt) { - initMobileColumnActions(evt.detail); -}); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js deleted file mode 100644 index ec2d23d8..00000000 --- a/datasette/static/navigation-search.js +++ /dev/null @@ -1,910 +0,0 @@ -let navigationSearchInstanceCounter = 0; - -class NavigationSearch extends HTMLElement { - constructor() { - super(); - this.instanceId = ++navigationSearchInstanceCounter; - this.inputId = `navigation-search-input-${this.instanceId}`; - this.instructionsId = `navigation-search-instructions-${this.instanceId}`; - this.listboxId = `navigation-search-results-${this.instanceId}`; - this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; - this.statusId = `navigation-search-status-${this.instanceId}`; - this.titleId = `navigation-search-title-${this.instanceId}`; - this.attachShadow({ mode: "open" }); - this.selectedIndex = -1; - this.matches = []; - this.renderedMatches = []; - this.debounceTimer = null; - this.restoreFocusTarget = null; - this.shouldRestoreFocus = true; - - this.render(); - this.setupEventListeners(); - } - - render() { - this.shadowRoot.innerHTML = ` - - - -
    - - - -
    - - -
    -
    -
    - ↑ ↓ Navigate - Enter Select - Esc Close -
    -
    -     - `; - } - - setupEventListeners() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const closeButton = this.shadowRoot.querySelector(".close-search"); - const resultsContainer = - this.shadowRoot.querySelector(".results-container"); - - // Global keyboard listener for "/" - document.addEventListener("keydown", (e) => { - if (e.key === "/" && !this.isInputFocused() && !dialog.open) { - e.preventDefault(); - this.openMenu(); - } - }); - - document.addEventListener("click", (e) => { - const trigger = e.target.closest("[data-navigation-search-open]"); - if (trigger) { - e.preventDefault(); - const details = trigger.closest("details"); - const restoreTarget = details?.querySelector("summary") || trigger; - details?.removeAttribute("open"); - this.openMenu(restoreTarget); - } - }); - - // Input event - input.addEventListener("input", (e) => { - this.handleSearch(e.target.value); - }); - - // Keyboard navigation - input.addEventListener("keydown", (e) => { - if (e.key === "ArrowDown") { - e.preventDefault(); - this.moveSelection(1); - } else if (e.key === "ArrowUp") { - e.preventDefault(); - this.moveSelection(-1); - } else if (e.key === "Enter") { - e.preventDefault(); - this.selectCurrentItem(); - } else if (e.key === "Escape") { - this.closeMenu(); - } - }); - - closeButton.addEventListener("click", () => { - this.closeMenu(); - }); - - // Click on result item - resultsContainer.addEventListener("click", (e) => { - const clearRecent = e.target.closest("[data-clear-recent-items]"); - if (clearRecent) { - e.preventDefault(); - this.clearRecentItems(); - return; - } - - const item = e.target.closest(".result-item"); - if (item) { - const index = parseInt(item.dataset.index); - this.selectItem(index); - } - }); - - // Close on backdrop click - dialog.addEventListener("click", (e) => { - if (e.target === dialog) { - this.closeMenu(); - } - }); - - dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this.closeMenu(); - }); - - dialog.addEventListener("close", () => { - this.onMenuClosed(); - }); - - // Initial load - this.loadInitialData(); - } - - isInputFocused() { - const activeElement = document.activeElement; - return ( - activeElement && - (activeElement.tagName === "INPUT" || - activeElement.tagName === "TEXTAREA" || - activeElement.isContentEditable) - ); - } - - setElementAttribute(element, name, value) { - if (!element) { - return; - } - if (typeof element.setAttribute === "function") { - element.setAttribute(name, value); - } else { - element[name] = String(value); - } - } - - removeElementAttribute(element, name) { - if (!element) { - return; - } - if (typeof element.removeAttribute === "function") { - element.removeAttribute(name); - } else { - delete element[name]; - } - } - - focusRestoreTarget(trigger) { - if (trigger && typeof trigger.focus === "function") { - return trigger; - } - if ( - document.activeElement && - typeof document.activeElement.focus === "function" - ) { - return document.activeElement; - } - return null; - } - - setNavigationTriggersExpanded(expanded) { - if (typeof document.querySelectorAll !== "function") { - return; - } - document - .querySelectorAll("[data-navigation-search-open]") - .forEach((trigger) => { - this.setElementAttribute( - trigger, - "aria-expanded", - expanded ? "true" : "false", - ); - }); - } - - resultOptionId(index) { - return `${this.listboxId}-option-${index}`; - } - - updateComboboxState() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const matches = this.renderedMatches || []; - this.setElementAttribute( - input, - "aria-expanded", - dialog && dialog.open && matches.length > 0 ? "true" : "false", - ); - - if ( - dialog && - dialog.open && - this.selectedIndex >= 0 && - this.selectedIndex < matches.length - ) { - this.setElementAttribute( - input, - "aria-activedescendant", - this.resultOptionId(this.selectedIndex), - ); - } else { - this.removeElementAttribute(input, "aria-activedescendant"); - } - } - - setStatus(message) { - const status = this.shadowRoot.querySelector(`#${this.statusId}`); - if (status) { - status.textContent = message || ""; - } - } - - resultsStatus(count, truncated) { - if (truncated) { - return "More than 100 results. Keep typing to narrow the list."; - } - if (count === 0) { - return "No results found."; - } - if (count === 1) { - return "1 result."; - } - return `${count} results.`; - } - - loadInitialData() { - const itemsAttr = this.getAttribute("items"); - if (itemsAttr) { - try { - this.allItems = JSON.parse(itemsAttr); - this.matches = this.allItems; - } catch (e) { - console.error("Failed to parse items attribute:", e); - this.allItems = []; - this.matches = []; - } - } - } - - handleSearch(query) { - clearTimeout(this.debounceTimer); - if (query.trim()) { - this.setStatus("Searching..."); - } else { - this.setStatus(""); - } - - this.debounceTimer = setTimeout(() => { - const url = this.getAttribute("url"); - - if (url) { - // Fetch from API - this.fetchResults(url, query); - } else { - // Filter local items - this.filterLocalItems(query); - } - }, 200); - } - - async fetchResults(url, query) { - try { - const searchUrl = `${url}?q=${encodeURIComponent(query)}`; - const response = await fetch(searchUrl); - const data = await response.json(); - this.matches = data.matches || []; - this.selectedIndex = this.matches.length > 0 ? 0 : -1; - this.renderResults(); - if (query.trim()) { - this.setStatus(this.resultsStatus(this.matches.length, data.truncated)); - } else { - this.setStatus(""); - } - } catch (e) { - console.error("Failed to fetch search results:", e); - this.matches = []; - this.renderResults(); - this.setStatus("Search failed."); - } - } - - filterLocalItems(query) { - if (!query.trim()) { - this.matches = this.allItems || []; - } else { - const lowerQuery = query.toLowerCase(); - this.matches = (this.allItems || []).filter( - (item) => - item.name.toLowerCase().includes(lowerQuery) || - (item.display_name || "").toLowerCase().includes(lowerQuery) || - item.url.toLowerCase().includes(lowerQuery), - ); - } - this.selectedIndex = this.matches.length > 0 ? 0 : -1; - this.renderResults(); - if (query.trim()) { - this.setStatus(this.resultsStatus(this.matches.length, false)); - } else { - this.setStatus(""); - } - } - - recentItemsStorageKey() { - return "datasette.navigationSearch.recentItems"; - } - - loadRecentItems() { - if (typeof localStorage === "undefined") { - return []; - } - - try { - const raw = localStorage.getItem(this.recentItemsStorageKey()); - if (!raw) { - return []; - } - const parsed = JSON.parse(raw); - if (!Array.isArray(parsed)) { - return []; - } - return parsed - .filter((item) => item && item.name && item.url) - .map((item) => ({ - name: String(item.name), - display_name: item.display_name ? String(item.display_name) : "", - url: String(item.url), - type: item.type ? String(item.type) : "", - description: item.description ? String(item.description) : "", - })) - .slice(0, 5); - } catch (e) { - return []; - } - } - - saveRecentItem(match) { - if ( - typeof localStorage === "undefined" || - !match || - !match.name || - !match.url - ) { - return; - } - - try { - const item = { - name: String(match.name), - display_name: match.display_name ? String(match.display_name) : "", - url: String(match.url), - type: match.type ? String(match.type) : "", - description: match.description ? String(match.description) : "", - }; - const recentItems = this.loadRecentItems().filter( - (recentItem) => recentItem.url !== item.url, - ); - localStorage.setItem( - this.recentItemsStorageKey(), - JSON.stringify([item, ...recentItems].slice(0, 5)), - ); - } catch (e) { - // localStorage may be unavailable, full, or disabled. - } - } - - clearRecentItems() { - if (typeof localStorage === "undefined") { - return; - } - - try { - localStorage.removeItem(this.recentItemsStorageKey()); - } catch (e) { - localStorage.setItem(this.recentItemsStorageKey(), "[]"); - } - this.renderResults(); - this.setStatus("Recent items cleared."); - } - - jumpSections() { - const manager = window.__DATASETTE__; - if (!manager || typeof manager.makeJumpSections !== "function") { - return []; - } - const sections = manager.makeJumpSections({ - navigationSearch: this, - }); - return Array.isArray(sections) - ? sections.filter( - (section) => section && typeof section.render === "function", - ) - : []; - } - - jumpSectionsHtml(jumpSections) { - return jumpSections - .map((section, index) => { - const id = section.id - ? ` data-jump-section-id="${this.escapeHtml(section.id)}"` - : ""; - return `
    `; - }) - .join(""); - } - - renderJumpSections(container, jumpSections) { - jumpSections.forEach((section, index) => { - const node = container.querySelector( - `[data-jump-section-index="${index}"]`, - ); - if (!node) { - return; - } - section.render(node, { - navigationSearch: this, - container, - input: this.shadowRoot.querySelector(".search-input"), - }); - }); - } - - resultItemHtml(match, index) { - const displayName = match.display_name || match.name; - const label = - match.display_name && match.display_name !== match.name - ? `
    ${this.escapeHtml(match.name)}
    ` - : ""; - const type = match.type - ? `
    ${this.escapeHtml(match.type)}
    ` - : ""; - const description = match.description - ? `
    ${this.escapeHtml( - match.description, - )}
    ` - : ""; - return ` -
    -
    - ${type} -
    ${this.escapeHtml(displayName)}
    - ${label} -
    ${this.escapeHtml(match.url)}
    - ${description} -
    -
    - `; - } - - renderResults() { - const container = this.shadowRoot.querySelector(".results-container"); - const input = this.shadowRoot.querySelector(".search-input"); - const showStartContent = !input.value.trim(); - const jumpSections = showStartContent ? this.jumpSections() : []; - const startBlock = showStartContent - ? this.jumpSectionsHtml(jumpSections) - : ""; - const recentItems = showStartContent ? this.loadRecentItems() : []; - const defaultMatches = showStartContent ? [] : this.matches; - const renderedMatches = [...recentItems, ...defaultMatches]; - this.renderedMatches = renderedMatches; - const emptyListbox = `
    `; - - if (renderedMatches.length) { - if ( - this.selectedIndex < 0 || - this.selectedIndex >= renderedMatches.length - ) { - this.selectedIndex = 0; - } - } else { - this.selectedIndex = -1; - } - - if (renderedMatches.length === 0) { - if (startBlock) { - container.innerHTML = startBlock + emptyListbox; - this.renderJumpSections(container, jumpSections); - } else if (showStartContent) { - container.innerHTML = emptyListbox; - } else { - const message = input.value.trim() - ? "No results found" - : "Start typing to search..."; - container.innerHTML = `${emptyListbox}
    ${message}
    `; - } - this.updateComboboxState(); - return; - } - - const recentHeading = recentItems.length - ? `
    Recent
    ` - : ""; - const recentGroup = recentItems.length - ? `
    ${recentItems - .map((match, index) => this.resultItemHtml(match, index)) - .join("")}
    ` - : ""; - const recentActions = recentItems.length - ? `
    ` - : ""; - const defaultHtml = defaultMatches - .map((match, index) => - this.resultItemHtml(match, recentItems.length + index), - ) - .join(""); - container.innerHTML = - startBlock + - recentHeading + - `
    ${recentGroup}${defaultHtml}
    ` + - recentActions; - this.renderJumpSections(container, jumpSections); - this.updateComboboxState(); - - // Scroll selected item into view - if (this.selectedIndex >= 0) { - const selectedItem = container.querySelector( - `.result-item[data-index="${this.selectedIndex}"]`, - ); - if (selectedItem) { - selectedItem.scrollIntoView({ block: "nearest" }); - } - } - } - - moveSelection(direction) { - const matches = this.renderedMatches || this.matches; - const newIndex = this.selectedIndex + direction; - if (newIndex >= 0 && newIndex < matches.length) { - this.selectedIndex = newIndex; - this.renderResults(); - } - } - - selectCurrentItem() { - const matches = this.renderedMatches || this.matches; - if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) { - this.selectItem(this.selectedIndex); - } - } - - selectItem(index) { - const matches = this.renderedMatches || this.matches; - const match = matches[index]; - if (match) { - this.saveRecentItem(match); - - // Dispatch custom event - this.dispatchEvent( - new CustomEvent("select", { - detail: match, - bubbles: true, - composed: true, - }), - ); - - // Navigate to URL - window.location.href = match.url; - - this.closeMenu({ restoreFocus: false }); - } - } - - openMenu(trigger) { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - - this.restoreFocusTarget = this.focusRestoreTarget(trigger); - this.shouldRestoreFocus = true; - if (!dialog.open) { - dialog.showModal(); - } - this.setNavigationTriggersExpanded(true); - input.value = ""; - input.focus(); - - // Reset state, then populate the default jump list. - this.matches = []; - this.selectedIndex = -1; - this.renderResults(); - this.setStatus(""); - } - - closeMenu(options = {}) { - const dialog = this.shadowRoot.querySelector("dialog"); - this.shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - this.onMenuClosed(); - } - } - - onMenuClosed() { - const input = this.shadowRoot.querySelector(".search-input"); - this.setElementAttribute(input, "aria-expanded", "false"); - this.removeElementAttribute(input, "aria-activedescendant"); - this.setNavigationTriggersExpanded(false); - this.setStatus(""); - if ( - this.shouldRestoreFocus && - this.restoreFocusTarget && - typeof this.restoreFocusTarget.focus === "function" - ) { - this.restoreFocusTarget.focus(); - } - this.restoreFocusTarget = null; - } - - escapeHtml(text) { - const div = document.createElement("div"); - div.textContent = text == null ? "" : text; - return div.innerHTML; - } -} - -// Register the custom element -customElements.define("navigation-search", NavigationSearch); diff --git a/datasette/static/table.js b/datasette/static/table.js index e9115453..51e901a5 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -1,6 +1,13 @@ var DROPDOWN_HTML = `
    -
      +

      `; @@ -10,518 +17,64 @@ var DROPDOWN_ICON_SVG = ` `; -var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog"; -var setColumnTypeDialogState = null; - -function getParams() { - return new URLSearchParams(location.search); -} - -function paramsToUrl(params) { - var s = params.toString(); - return s ? "?" + s : location.pathname; -} - -function sortDescUrl(column) { - var params = getParams(); - params.set("_sort_desc", column); - params.delete("_sort"); - params.delete("_next"); - return paramsToUrl(params); -} - -function sortAscUrl(column) { - var params = getParams(); - params.set("_sort", column); - params.delete("_sort_desc"); - params.delete("_next"); - return paramsToUrl(params); -} - -function facetUrl(column) { - var params = getParams(); - params.append("_facet", column); - return paramsToUrl(params); -} - -function hideColumnUrl(column) { - var params = getParams(); - params.append("_nocol", column); - return paramsToUrl(params); -} - -function showAllColumnsUrl() { - var params = getParams(); - params.delete("_nocol"); - params.delete("_col"); - return paramsToUrl(params); -} - -function notBlankUrl(column) { - var params = getParams(); - params.set(`${column}__notblank`, "1"); - return paramsToUrl(params); -} - -function getDisplayedFacets() { - return Array.from(document.querySelectorAll(".facet-info")).map( - (el) => el.dataset.column, - ); -} - -function getColumnClassName(th) { - return Array.from(th.classList).find((className) => - className.startsWith("col-"), - ); -} - -function getColumnCells(th) { - var table = th.closest("table"); - var columnClassName = getColumnClassName(th); - if (!table || !columnClassName) { - return []; - } - return Array.from(table.querySelectorAll("td." + columnClassName)); -} - -function getColumnMeta(th) { - return { - columnName: th.dataset.column, - columnNotNull: th.dataset.columnNotNull === "1", - columnType: th.dataset.columnType, - isPk: th.dataset.isPk === "1", - }; -} - -function getColumnTypeText(th) { - var columnType = th.dataset.columnType; - if (!columnType) { - return null; - } - var notNull = th.dataset.columnNotNull === "1" ? " NOT NULL" : ""; - return `Type: ${columnType.toUpperCase()}${notNull}`; -} - -function getSetColumnTypeData() { - return window._setColumnTypeData || null; -} - -function getSetColumnTypeConfig(column) { - var data = getSetColumnTypeData(); - if (!data || !data.columns) { - return null; - } - return data.columns[column] || null; -} - -function canSetColumnType() { - return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch); -} - -function setColumnTypeActionLabel(column) { - var columnConfig = getSetColumnTypeConfig(column); - if (!columnConfig) { - return null; - } - return columnConfig.current - ? `Custom type: ${columnConfig.current.type}` - : "Set custom type"; -} - -function createSetColumnTypeOption(value, name, description, checked) { - var label = document.createElement("label"); - label.className = "set-column-type-option"; - - var input = document.createElement("input"); - input.type = "radio"; - input.name = "set-column-type-choice"; - input.value = value; - input.checked = checked; - - var content = document.createElement("span"); - content.className = "set-column-type-option-content"; - - var title = document.createElement("span"); - title.className = "set-column-type-option-name"; - title.textContent = name; - - var detail = document.createElement("span"); - detail.className = "set-column-type-option-description"; - detail.textContent = description; - - content.appendChild(title); - content.appendChild(detail); - label.appendChild(input); - label.appendChild(content); - return label; -} - -function setSetColumnTypeDialogBusy(state, isBusy) { - state.isBusy = isBusy; - state.saveButton.disabled = isBusy; - state.cancelButton.disabled = isBusy; - Array.from( - state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'), - ).forEach(function (input) { - input.disabled = isBusy; - }); - state.saveButton.textContent = isBusy ? "Saving..." : "Save"; -} - -function clearSetColumnTypeDialogError(state) { - state.error.hidden = true; - state.error.textContent = ""; -} - -function showSetColumnTypeDialogError(state, message) { - state.error.hidden = false; - state.error.textContent = message; -} - -function ensureSetColumnTypeDialog() { - if (setColumnTypeDialogState) { - return setColumnTypeDialogState; - } - if (!window.HTMLDialogElement) { - return null; - } - - var dialog = document.createElement("dialog"); - dialog.id = SET_COLUMN_TYPE_DIALOG_ID; - dialog.className = "set-column-type-dialog"; - dialog.setAttribute("aria-labelledby", "set-column-type-title"); - dialog.innerHTML = ` -
      - Set custom type - -
      -

      - -
      -
      - - - -
      - `; - document.body.appendChild(dialog); - - setColumnTypeDialogState = { - dialog: dialog, - meta: dialog.querySelector(".modal-meta"), - status: dialog.querySelector(".set-column-type-status"), - error: dialog.querySelector(".set-column-type-error"), - optionsWrap: dialog.querySelector(".set-column-type-options"), - footerInfo: dialog.querySelector(".footer-info"), - cancelButton: dialog.querySelector(".set-column-type-cancel"), - saveButton: dialog.querySelector(".set-column-type-save"), - currentColumn: null, - currentConfig: null, - isBusy: false, - }; - - setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - if (!setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - if (setColumnTypeDialogState.isBusy) { - ev.preventDefault(); - } - }); - - dialog.addEventListener("close", function () { - clearSetColumnTypeDialogError(setColumnTypeDialogState); - setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); - }); - - setColumnTypeDialogState.saveButton.addEventListener("click", async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; - - if (selectedType === currentType) { - state.dialog.close(); - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); - } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }); - - return setColumnTypeDialogState; -} - -function openSetColumnTypeDialog(th) { - var column = th.dataset.column; - var columnConfig = getSetColumnTypeConfig(column); - if (!columnConfig) { - return; - } - - var state = ensureSetColumnTypeDialog(); - if (!state) { - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, false); - state.currentColumn = column; - state.currentConfig = columnConfig; - state.status.textContent = `Column: ${column}`; - state.meta.textContent = getColumnTypeText(th) || "Type unavailable"; - state.footerInfo.textContent = columnConfig.current - ? `Current custom type: ${columnConfig.current.type}` - : "No custom type set."; - state.optionsWrap.innerHTML = ""; - - var currentType = columnConfig.current ? columnConfig.current.type : ""; - state.optionsWrap.appendChild( - createSetColumnTypeOption( - "", - "No custom type", - "Use standard Datasette rendering without a custom type.", - currentType === "", - ), - ); - - columnConfig.options.forEach(function (option) { - state.optionsWrap.appendChild( - createSetColumnTypeOption( - option.name, - option.name, - option.description, - option.name === currentType, - ), - ); - }); - - if (!columnConfig.options.length) { - var emptyState = document.createElement("p"); - emptyState.className = "set-column-type-empty"; - emptyState.textContent = - "No registered custom types are compatible with this SQLite type."; - state.optionsWrap.appendChild(emptyState); - } - - if (!state.dialog.open) { - state.dialog.showModal(); - } - var selectedOption = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - if (selectedOption) { - selectedOption.focus(); - } else { - state.saveButton.focus(); - } -} - -function canChooseColumns() { - return !!( - document.querySelector("column-chooser") && window._columnChooserData - ); -} - -function shouldShowShowAllColumns() { - var params = getParams(); - return params.getAll("_nocol").length || params.getAll("_col").length; -} - -function hasMultipleVisibleColumns(manager) { - return ( - Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( - (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", - ).length > 1 - ); -} - -function buildColumnActionItems(manager, th, options) { - options = options || {}; - var params = getParams(); - var column = th.dataset.column; - var columnActions = []; - var isSortable = !!th.querySelector("a"); - var isFirstColumn = th.parentElement.querySelector("th:first-of-type") === th; - var isSinglePk = - th.dataset.isPk === "1" && - document.querySelectorAll('th[data-is-pk="1"]').length === 1; - var hasBlankValues = getColumnCells(th).some( - (el) => el.innerText.trim() === "", - ); - - if (isSortable && params.get("_sort") !== column) { - columnActions.push({ - label: "Sort ascending", - href: sortAscUrl(column), - }); - } - - if (isSortable && params.get("_sort_desc") !== column) { - columnActions.push({ - label: "Sort descending", - href: sortDescUrl(column), - }); - } - - if ( - DATASETTE_ALLOW_FACET && - !isFirstColumn && - !getDisplayedFacets().includes(column) && - !isSinglePk - ) { - columnActions.push({ - label: "Facet by this", - href: facetUrl(column), - }); - } - - if (options.includeChooseColumns && canChooseColumns()) { - columnActions.push({ - label: "Choose columns", - href: "#", - onClick: - options.onChooseColumns || - function (ev) { - ev.preventDefault(); - openColumnChooser(); - }, - }); - } - - if (canSetColumnType() && getSetColumnTypeConfig(column)) { - columnActions.push({ - label: setColumnTypeActionLabel(column), - href: "#", - onClick: - options.onSetColumnType || - function (ev) { - ev.preventDefault(); - window.setTimeout(function () { - openSetColumnTypeDialog(th); - }, 0); - }, - }); - } - - if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) { - columnActions.push({ - label: "Hide this column", - href: hideColumnUrl(column), - }); - } - - if (options.includeShowAllColumns && shouldShowShowAllColumns()) { - columnActions.push({ - label: "Show all columns", - href: showAllColumnsUrl(), - }); - } - - if (params.get(`${column}__notblank`) !== "1" && hasBlankValues) { - columnActions.push({ - label: "Show not-blank rows", - href: notBlankUrl(column), - }); - } - - return columnActions.concat(manager.makeColumnActions(getColumnMeta(th))); -} - -function buildColumnActionState(manager, th, options) { - return { - column: th.dataset.column, - columnDescription: th.dataset.columnDescription || null, - columnMeta: getColumnMeta(th), - columnTypeText: getColumnTypeText(th), - actionItems: buildColumnActionItems(manager, th, options), - }; -} - -function initializeColumnActions(manager) { - manager.columnActions = { - buildColumnActionState: function (th, options) { - return buildColumnActionState(manager, th, options); - }, - buildColumnActionItems: function (th, options) { - return buildColumnActionItems(manager, th, options); - }, - canChooseColumns: canChooseColumns, - facetUrl: facetUrl, - getColumnMeta: getColumnMeta, - getColumnTypeText: getColumnTypeText, - hideColumnUrl: hideColumnUrl, - notBlankUrl: notBlankUrl, - shouldShowShowAllColumns: shouldShowShowAllColumns, - showAllColumnsUrl: showAllColumnsUrl, - sortAscUrl: sortAscUrl, - sortDescUrl: sortDescUrl, - }; -} - -function renderActionLink(itemConfig) { - var newLink = document.createElement("a"); - newLink.textContent = itemConfig.label; - newLink.href = itemConfig.href || "#"; - if (itemConfig.onClick) { - newLink.addEventListener("click", itemConfig.onClick); - } - return newLink; -} - -/** Main initialization function for Datasette Table interactions */ -const initDatasetteTable = function (manager) { +(function () { // Feature detection if (!window.URLSearchParams) { return; } + function getParams() { + return new URLSearchParams(location.search); + } + function paramsToUrl(params) { + var s = params.toString(); + return s ? "?" + s : location.pathname; + } + function sortDescUrl(column) { + var params = getParams(); + params.set("_sort_desc", column); + params.delete("_sort"); + params.delete("_next"); + return paramsToUrl(params); + } + function sortAscUrl(column) { + var params = getParams(); + params.set("_sort", column); + params.delete("_sort_desc"); + params.delete("_next"); + return paramsToUrl(params); + } + function facetUrl(column) { + var params = getParams(); + params.append("_facet", column); + return paramsToUrl(params); + } + function hideColumnUrl(column) { + var params = getParams(); + params.append("_nocol", column); + return paramsToUrl(params); + } + function showAllColumnsUrl() { + var params = getParams(); + params.delete("_nocol"); + params.delete("_col"); + return paramsToUrl(params); + } + function notBlankUrl(column) { + var params = getParams(); + params.set(`${column}__notblank`, "1"); + return paramsToUrl(params); + } function closeMenu() { menu.style.display = "none"; menu.classList.remove("anim-scale-in"); } - - const tableWrapper = document.querySelector(manager.selectors.tableWrapper); - if (tableWrapper) { - tableWrapper.addEventListener("scroll", closeMenu); - } + // When page loads, add scroll listener on .table-wrapper + document.addEventListener("DOMContentLoaded", () => { + var tableWrapper = document.querySelector(".table-wrapper"); + if (tableWrapper) { + tableWrapper.addEventListener("scroll", closeMenu); + } + }); document.body.addEventListener("click", (ev) => { /* was this click outside the menu? */ var target = ev.target; @@ -532,11 +85,9 @@ const initDatasetteTable = function (manager) { closeMenu(); } }); - - function onTableHeaderClick(ev) { + function iconClicked(ev) { ev.preventDefault(); ev.stopPropagation(); - menu.innerHTML = DROPDOWN_HTML; var th = ev.target; while (th.nodeName != "TH") { th = th.parentNode; @@ -544,41 +95,87 @@ const initDatasetteTable = function (manager) { var rect = th.getBoundingClientRect(); var menuTop = rect.bottom + window.scrollY; var menuLeft = rect.left + window.scrollX; - var actionState = manager.columnActions.buildColumnActionState(th, { - includeChooseColumns: true, - includeShowAllColumns: true, - onChooseColumns: function (ev) { - ev.preventDefault(); - closeMenu(); - openColumnChooser(); - }, - onSetColumnType: function (ev) { - ev.preventDefault(); - closeMenu(); - window.setTimeout(function () { - openSetColumnTypeDialog(th); - }, 0); - }, - }); - var menuList = menu.querySelector("ul.dropdown-actions"); - menuList.innerHTML = ""; - actionState.actionItems.forEach((itemConfig) => { - var menuItem = document.createElement("li"); - menuItem.appendChild(renderActionLink(itemConfig)); - menuList.appendChild(menuItem); - }); - + var column = th.getAttribute("data-column"); + var params = getParams(); + var sort = menu.querySelector("a.dropdown-sort-asc"); + var sortDesc = menu.querySelector("a.dropdown-sort-desc"); + var facetItem = menu.querySelector("a.dropdown-facet"); + var notBlank = menu.querySelector("a.dropdown-not-blank"); + var hideColumn = menu.querySelector("a.dropdown-hide-column"); + var showAllColumns = menu.querySelector("a.dropdown-show-all-columns"); + if (params.get("_sort") == column) { + sort.parentNode.style.display = "none"; + } else { + sort.parentNode.style.display = "block"; + sort.setAttribute("href", sortAscUrl(column)); + } + if (params.get("_sort_desc") == column) { + sortDesc.parentNode.style.display = "none"; + } else { + sortDesc.parentNode.style.display = "block"; + sortDesc.setAttribute("href", sortDescUrl(column)); + } + /* Show hide columns options */ + if (params.get("_nocol") || params.get("_col")) { + showAllColumns.parentNode.style.display = "block"; + showAllColumns.setAttribute("href", showAllColumnsUrl()); + } else { + showAllColumns.parentNode.style.display = "none"; + } + if (th.getAttribute("data-is-pk") != "1") { + hideColumn.parentNode.style.display = "block"; + hideColumn.setAttribute("href", hideColumnUrl(column)); + } else { + hideColumn.parentNode.style.display = "none"; + } + /* Only show "Facet by this" if it's not the first column, not selected, + not a single PK and the Datasette allow_facet setting is True */ + var displayedFacets = Array.from( + document.querySelectorAll(".facet-info") + ).map((el) => el.dataset.column); + var isFirstColumn = + th.parentElement.querySelector("th:first-of-type") == th; + var isSinglePk = + th.getAttribute("data-is-pk") == "1" && + document.querySelectorAll('th[data-is-pk="1"]').length == 1; + if ( + !DATASETTE_ALLOW_FACET || + isFirstColumn || + displayedFacets.includes(column) || + isSinglePk + ) { + facetItem.parentNode.style.display = "none"; + } else { + facetItem.parentNode.style.display = "block"; + facetItem.setAttribute("href", facetUrl(column)); + } + /* Show notBlank option if not selected AND at least one visible blank value */ + var tdsForThisColumn = Array.from( + th.closest("table").querySelectorAll("td." + th.className) + ); + if ( + params.get(`${column}__notblank`) != "1" && + tdsForThisColumn.filter((el) => el.innerText.trim() == "").length + ) { + notBlank.parentNode.style.display = "block"; + notBlank.setAttribute("href", notBlankUrl(column)); + } else { + notBlank.parentNode.style.display = "none"; + } var columnTypeP = menu.querySelector(".dropdown-column-type"); - if (actionState.columnTypeText) { + var columnType = th.dataset.columnType; + var notNull = th.dataset.columnNotNull == 1 ? " NOT NULL" : ""; + + if (columnType) { columnTypeP.style.display = "block"; - columnTypeP.innerText = actionState.columnTypeText; + columnTypeP.innerText = `Type: ${columnType.toUpperCase()}${notNull}`; } else { columnTypeP.style.display = "none"; } var columnDescriptionP = menu.querySelector(".dropdown-column-description"); - if (actionState.columnDescription) { - columnDescriptionP.innerText = actionState.columnDescription; + if (th.dataset.columnDescription) { + columnDescriptionP.innerText = th.dataset.columnDescription; columnDescriptionP.style.display = "block"; } else { columnDescriptionP.style.display = "none"; @@ -588,28 +185,7 @@ const initDatasetteTable = function (manager) { menu.style.left = menuLeft + "px"; menu.style.display = "block"; menu.classList.add("anim-scale-in"); - - // Measure width of menu and adjust position if too far right - const menuWidth = menu.offsetWidth; - const windowWidth = window.innerWidth; - if (menuLeft + menuWidth > windowWidth) { - menu.style.left = windowWidth - menuWidth - 20 + "px"; - } - // Align menu .hook arrow with the column cog icon - const hook = menu.querySelector(".hook"); - const icon = th.querySelector(".dropdown-menu-icon"); - const iconRect = icon.getBoundingClientRect(); - const hookLeft = iconRect.left - menuLeft + 1 + "px"; - hook.style.left = hookLeft; - // Move the whole menu right if the hook is too far right - const menuRect = menu.getBoundingClientRect(); - if (iconRect.right > menuRect.right) { - menu.style.left = iconRect.right - menuWidth + "px"; - // And move hook tip as well - hook.style.left = menuWidth - 13 + "px"; - } } - var svg = document.createElement("div"); svg.innerHTML = DROPDOWN_ICON_SVG; svg = svg.querySelector("*"); @@ -621,25 +197,23 @@ const initDatasetteTable = function (manager) { menu.style.display = "none"; document.body.appendChild(menu); - var ths = Array.from( - document.querySelectorAll(manager.selectors.tableHeaders), - ); + var ths = Array.from(document.querySelectorAll(".rows-and-columns th")); ths.forEach((th) => { if (!th.querySelector("a")) { return; } var icon = svg.cloneNode(true); - icon.addEventListener("click", onTableHeaderClick); + icon.addEventListener("click", iconClicked); th.appendChild(icon); }); -}; +})(); /* Add x buttons to the filter rows */ -function addButtonsToFilterRows(manager) { +(function () { var x = "✖"; - var rows = Array.from( - document.querySelectorAll(manager.selectors.filterRow), - ).filter((el) => el.querySelector(".filter-op")); + var rows = Array.from(document.querySelectorAll(".filter-row")).filter((el) => + el.querySelector(".filter-op") + ); rows.forEach((row) => { var a = document.createElement("a"); a.setAttribute("href", "#"); @@ -660,18 +234,18 @@ function addButtonsToFilterRows(manager) { a.style.display = "none"; } }); -} +})(); /* Set up datalist autocomplete for filter values */ -function initAutocompleteForFilterValues(manager) { +(function () { function createDataLists() { var facetResults = document.querySelectorAll( - manager.selectors.facetResults, + ".facet-results [data-column]" ); Array.from(facetResults).forEach(function (facetResult) { // Use link text from all links in the facet result var links = Array.from( - facetResult.querySelectorAll("li:not(.facet-truncated) a"), + facetResult.querySelectorAll("li:not(.facet-truncated) a") ); // Create a datalist element var datalist = document.createElement("datalist"); @@ -692,66 +266,9 @@ function initAutocompleteForFilterValues(manager) { document.body.addEventListener("change", function (event) { if (event.target.name === "_filter_column") { event.target - .closest(manager.selectors.filterRow) + .closest(".filter-row") .querySelector(".filter-value") .setAttribute("list", "datalist-" + event.target.value); } }); -} - -/** Open the column-chooser web component */ -function openColumnChooser() { - var chooser = document.querySelector("column-chooser"); - var data = window._columnChooserData; - if (!chooser || !data) return; - - var nonPkColumns = data.allColumns.filter(function (col) { - return data.primaryKeys.indexOf(col) === -1; - }); - var selected = data.selectedColumns.filter(function (col) { - return data.primaryKeys.indexOf(col) === -1; - }); - - chooser.open({ - columns: nonPkColumns, - selected: selected, - onApply: function (cols) { - var params = new URLSearchParams(location.search); - params.delete("_col"); - params.delete("_nocol"); - params.delete("_next"); - - if (cols.length === nonPkColumns.length) { - // Check if order matches original - if so, no params needed - var orderMatches = cols.every(function (col, i) { - return col === nonPkColumns[i]; - }); - if (!orderMatches) { - cols.forEach(function (col) { - params.append("_col", col); - }); - } - } else { - cols.forEach(function (col) { - params.append("_col", col); - }); - } - var qs = params.toString(); - location.href = qs ? "?" + qs : location.pathname; - }, - }); -} - -// Ensures Table UI is initialized only after the Manager is ready. -document.addEventListener("datasette_init", function (evt) { - const { detail: manager } = evt; - - initializeColumnActions(manager); - - // Main table - initDatasetteTable(manager); - - // Other UI functions with interactive JS needs - addButtonsToFilterRows(manager); - initAutocompleteForFilterValues(manager); -}); +})(); diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py deleted file mode 100644 index bcfdfdb4..00000000 --- a/datasette/stored_queries.py +++ /dev/null @@ -1,623 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -from typing import Any, Iterable - -from .resources import TableResource -from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components -from .utils.asgi import Forbidden - -UNCHANGED = object() - -QUERY_OPTION_FIELDS = ( - "hide_sql", - "fragment", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", -) - - -@dataclass -class StoredQuery: - database: str - name: str - sql: str - title: str | None - description: str | None - description_html: str | None - hide_sql: bool - fragment: str | None - parameters: list[str] - is_write: bool - is_private: bool - is_trusted: bool - source: str - owner_id: str | None - on_success_message: str | None - on_success_message_sql: str | None - on_success_redirect: str | None - on_error_message: str | None - on_error_redirect: str | None - private: bool | None = None - - -@dataclass -class StoredQueryPage: - queries: list[StoredQuery] - next: str | None - has_more: bool - limit: int - - -def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: - data = { - "database": query.database, - "name": query.name, - "sql": query.sql, - "title": query.title, - "description": query.description, - "description_html": query.description_html, - "hide_sql": query.hide_sql, - "fragment": query.fragment, - "params": list(query.parameters), - "parameters": list(query.parameters), - "is_write": query.is_write, - "is_private": query.is_private, - "is_trusted": query.is_trusted, - "source": query.source, - "owner_id": query.owner_id, - "on_success_message": query.on_success_message, - "on_success_message_sql": query.on_success_message_sql, - "on_success_redirect": query.on_success_redirect, - "on_error_message": query.on_error_message, - "on_error_redirect": query.on_error_redirect, - } - if query.private is not None: - data["private"] = query.private - return data - - -def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: - return { - "queries": [stored_query_to_dict(query) for query in page.queries], - "next": page.next, - "has_more": page.has_more, - "limit": page.limit, - } - - -async def save_queries_from_config(datasette: Any) -> None: - # Apply configured query entries from datasette.yaml to the internal table. - await datasette.get_internal_database().execute_write( - "DELETE FROM queries WHERE source = 'config'" - ) - for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if not isinstance(query_config, dict): - query_config = {"sql": query_config} - await datasette.add_query( - dbname, - query_name, - query_config["sql"], - title=query_config.get("title"), - description=query_config.get("description"), - description_html=query_config.get("description_html"), - hide_sql=bool(query_config.get("hide_sql")), - fragment=query_config.get("fragment"), - parameters=query_config.get("params"), - is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), - is_trusted=bool(query_config.get("is_trusted", True)), - source="config", - on_success_message=query_config.get("on_success_message"), - on_success_message_sql=query_config.get("on_success_message_sql"), - on_success_redirect=query_config.get("on_success_redirect"), - on_error_message=query_config.get("on_error_message"), - on_error_redirect=query_config.get("on_error_redirect"), - ) - - -def query_row_to_stored_query( - row: Any, private: bool | None = None -) -> StoredQuery | None: - if row is None: - return None - parameters = json.loads(row["parameters"] or "[]") - options = json.loads(row["options"] or "{}") - return StoredQuery( - database=row["database_name"], - name=row["name"], - sql=row["sql"], - title=row["title"], - description=row["description"], - description_html=row["description_html"], - hide_sql=bool(options.get("hide_sql")), - fragment=options.get("fragment"), - parameters=parameters, - is_write=bool(row["is_write"]), - is_private=bool(row["is_private"]), - is_trusted=bool(row["is_trusted"]), - source=row["source"], - owner_id=row["owner_id"], - on_success_message=options.get("on_success_message"), - on_success_message_sql=options.get("on_success_message_sql"), - on_success_redirect=options.get("on_success_redirect"), - on_error_message=options.get("on_error_message"), - on_error_redirect=options.get("on_error_redirect"), - private=private, - ) - - -def query_options_json(options: dict[str, Any]) -> str: - options_dict = {} - for field in QUERY_OPTION_FIELDS: - value = options.get(field) - if field == "hide_sql": - if value: - options_dict[field] = True - elif value is not None: - options_dict[field] = value - return json.dumps(options_dict, sort_keys=True) - - -async def add_query( - datasette: Any, - database: str, - name: str, - sql: str, - *, - title: str | None = None, - description: str | None = None, - description_html: str | None = None, - hide_sql: bool = False, - fragment: str | None = None, - parameters: Iterable[str] | None = None, - is_write: bool = False, - is_private: bool = False, - is_trusted: bool = False, - source: str = "plugin", - owner_id: str | None = None, - on_success_message: str | None = None, - on_success_message_sql: str | None = None, - on_success_redirect: str | None = None, - on_error_message: str | None = None, - on_error_redirect: str | None = None, - replace: bool = True, -) -> None: - parameters_json = json.dumps(list(parameters or [])) - options_json = query_options_json( - { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - ) - sql_statement = """ - INSERT INTO queries ( - database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_private, is_trusted, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - if replace: - sql_statement += """ - ON CONFLICT(database_name, name) DO UPDATE SET - sql = excluded.sql, - title = excluded.title, - description = excluded.description, - description_html = excluded.description_html, - options = excluded.options, - parameters = excluded.parameters, - is_write = excluded.is_write, - is_private = excluded.is_private, - is_trusted = excluded.is_trusted, - source = excluded.source, - owner_id = excluded.owner_id, - updated_at = CURRENT_TIMESTAMP - """ - await datasette.get_internal_database().execute_write( - sql_statement, - [ - database, - name, - sql, - title, - description, - description_html, - options_json, - parameters_json, - int(bool(is_write)), - int(bool(is_private)), - int(bool(is_trusted)), - source, - owner_id, - ], - ) - - -async def update_query( - datasette: Any, - database: str, - name: str, - *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, -) -> None: - fields = { - "sql": sql, - "title": title, - "description": description, - "description_html": description_html, - "parameters": parameters, - "is_write": is_write, - "is_private": is_private, - "is_trusted": is_trusted, - "source": source, - "owner_id": owner_id, - } - option_fields = { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - updates = [] - params = [] - for field, value in fields.items(): - if value is UNCHANGED: - continue - if field in {"is_write", "is_private", "is_trusted"}: - value = int(bool(value)) - elif field == "parameters": - value = json.dumps(list(value or [])) - updates.append(f"{field} = ?") - params.append(value) - changed_options = { - field: value for field, value in option_fields.items() if value is not UNCHANGED - } - if changed_options: - rows = await datasette.get_internal_database().execute( - """ - SELECT options FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - row = rows.first() - options = json.loads(row["options"] or "{}") if row is not None else {} - for field, value in changed_options.items(): - if field == "hide_sql": - if value: - options[field] = True - else: - options.pop(field, None) - elif value is None: - options.pop(field, None) - else: - options[field] = value - updates.append("options = ?") - params.append(json.dumps(options, sort_keys=True)) - if not updates: - return - updates.append("updated_at = CURRENT_TIMESTAMP") - params.extend([database, name]) - await datasette.get_internal_database().execute_write( - """ - UPDATE queries - SET {} - WHERE database_name = ? AND name = ? - """.format(", ".join(updates)), - params, - ) - - -async def remove_query( - datasette: Any, database: str, name: str, source: str | None = None -) -> None: - sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" - params = [database, name] - if source is not None: - sql += " AND source = ?" - params.append(source) - await datasette.get_internal_database().execute_write(sql, params) - - -async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: - rows = await datasette.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - return query_row_to_stored_query(rows.first()) - - -async def count_queries( - datasette: Any, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, -) -> int: - allowed_sql, allowed_params = await datasette.allowed_resources_sql( - action="view-query", - actor=actor, - parent=database, - ) - params = dict(allowed_params) - where_clauses = [] - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - row = ( - await datasette.get_internal_database().execute( - """ - SELECT count(*) AS count - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - """.format( - allowed_sql=allowed_sql, - where=" AND ".join(where_clauses) or "1 = 1", - ), - params, - ) - ).first() - return row["count"] - - -async def list_queries( - datasette: Any, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - limit: int = 50, - cursor: str | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - include_private: bool = False, -) -> StoredQueryPage: - limit = min(max(1, int(limit)), 1000) - allowed_sql, allowed_params = await datasette.allowed_resources_sql( - action="view-query", - actor=actor, - parent=database, - include_is_private=include_private, - ) - params = dict(allowed_params) - params.update({"limit": limit + 1}) - sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = [] - order_by = "q.database_name, sort_key, q.name" - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - order_by = "sort_key, q.name" - - if cursor: - try: - components = urlsafe_components(cursor) - except ValueError: - components = [] - if database is None and len(components) == 3: - where_clauses.append(""" - ( - q.database_name > :cursor_database - OR ( - q.database_name = :cursor_database - AND ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_database"] = components[0] - params["cursor_sort_key"] = components[1] - params["cursor_name"] = components[2] - elif database is not None and len(components) == 2: - where_clauses.append(""" - ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_sort_key"] = components[0] - params["cursor_name"] = components[1] - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - private_select = ", allowed.is_private AS private" if include_private else "" - rows = list( - ( - await datasette.get_internal_database().execute( - """ - SELECT q.*, {sort_key_sql} AS sort_key{private_select} - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - ORDER BY {order_by} - LIMIT :limit - """.format( - allowed_sql=allowed_sql, - private_select=private_select, - sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses) or "1 = 1", - order_by=order_by, - ), - params, - ) - ).rows - ) - has_more = len(rows) > limit - if has_more: - rows = rows[:limit] - - queries = [] - for row in rows: - query = query_row_to_stored_query( - row, private=bool(row["private"]) if include_private else None - ) - assert query is not None - queries.append(query) - - next_token = None - if has_more and rows: - last_row = rows[-1] - if database is None: - next_token = "{},{},{}".format( - tilde_encode(last_row["database_name"]), - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - else: - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - return StoredQueryPage( - queries=queries, - next=next_token, - has_more=has_more, - limit=limit, - ) - - -async def ensure_query_write_permissions( - datasette: Any, - database: str, - sql: str, - *, - actor: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - analysis: Any = None, -) -> Any: - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - db = datasette.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: - continue - if access.database != database: - raise Forbidden("Writable queries may not write to attached databases") - if not await datasette.allowed( - action=action, - resource=TableResource(database=access.database, table=access.table), - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) - return analysis diff --git a/datasette/templates/_action_menu.html b/datasette/templates/_action_menu.html deleted file mode 100644 index 1ae8c173..00000000 --- a/datasette/templates/_action_menu.html +++ /dev/null @@ -1,28 +0,0 @@ -{% if action_links %} -
      -
      - -
      -
      - -
      -
      -
      -{% endif %} \ No newline at end of file diff --git a/datasette/templates/_close_open_menus.html b/datasette/templates/_close_open_menus.html index a24633d0..3302d77d 100644 --- a/datasette/templates/_close_open_menus.html +++ b/datasette/templates/_close_open_menus.html @@ -13,50 +13,4 @@ document.body.addEventListener('click', (ev) => { (details) => details.open && details != detailsClickedWithin ).forEach(details => details.open = false); }); - -/* Sync aria-expanded and add keyboard navigation for details-menu elements */ -document.querySelectorAll('details.details-menu').forEach(function(details) { - var summary = details.querySelector('summary'); - details.addEventListener('toggle', function() { - if (summary) { - summary.setAttribute('aria-expanded', details.open ? 'true' : 'false'); - } - if (details.open) { - /* Focus first menu item when menu opens */ - var firstItem = details.querySelector('[role="menuitem"]'); - if (firstItem) { firstItem.focus(); } - } - }); -}); - -document.body.addEventListener('keydown', function(ev) { - /* Keyboard navigation for open details-menu elements */ - var openDetails = Array.from(document.querySelectorAll('details.details-menu[open]')); - if (!openDetails.length) { return; } - - if (ev.key === 'Escape') { - openDetails.forEach(function(details) { - details.open = false; - var summary = details.querySelector('summary'); - if (summary) { summary.focus(); } - }); - return; - } - - if (ev.key === 'ArrowDown' || ev.key === 'ArrowUp') { - var focused = document.activeElement; - openDetails.forEach(function(details) { - var items = Array.from(details.querySelectorAll('[role="menuitem"]')); - if (!items.length) { return; } - var idx = items.indexOf(focused); - if (idx === -1) { return; } - ev.preventDefault(); - if (ev.key === 'ArrowDown') { - items[(idx + 1) % items.length].focus(); - } else { - items[(idx - 1 + items.length) % items.length].focus(); - } - }); - } -}); diff --git a/datasette/templates/_debug_common_functions.html b/datasette/templates/_debug_common_functions.html deleted file mode 100644 index d988a2f3..00000000 --- a/datasette/templates/_debug_common_functions.html +++ /dev/null @@ -1,50 +0,0 @@ - diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html deleted file mode 100644 index a19bae13..00000000 --- a/datasette/templates/_execute_write_analysis_scripts.html +++ /dev/null @@ -1,111 +0,0 @@ - diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html deleted file mode 100644 index 165cfe9f..00000000 --- a/datasette/templates/_execute_write_analysis_styles.html +++ /dev/null @@ -1,41 +0,0 @@ - diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html deleted file mode 100644 index 53a824f1..00000000 --- a/datasette/templates/_permission_ui_styles.html +++ /dev/null @@ -1,145 +0,0 @@ - diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html deleted file mode 100644 index d7203c1e..00000000 --- a/datasette/templates/_permissions_debug_tabs.html +++ /dev/null @@ -1,54 +0,0 @@ -{% if has_debug_permission %} -{% set query_string = '?' + request.query_string if request.query_string else '' %} - - - - -{% endif %} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html deleted file mode 100644 index 159a141c..00000000 --- a/datasette/templates/_sql_parameter_scripts.html +++ /dev/null @@ -1,293 +0,0 @@ - diff --git a/datasette/templates/_sql_parameter_styles.html b/datasette/templates/_sql_parameter_styles.html deleted file mode 100644 index bc6838f5..00000000 --- a/datasette/templates/_sql_parameter_styles.html +++ /dev/null @@ -1,58 +0,0 @@ - diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html deleted file mode 100644 index 58801d40..00000000 --- a/datasette/templates/_sql_parameters.html +++ /dev/null @@ -1,9 +0,0 @@ -
      - {% if parameter_names %} -

      Parameters

      - {% for parameter in parameter_names %} - {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} -

      {% if sql_parameters_allow_expand|default(false) %} {% endif %}

      - {% endfor %} - {% endif %} -
      diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index f47a325f..5332f831 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,12 +1,10 @@ - -
      -{% if display_columns %} +{% if display_rows %}
      {% for column in display_columns %} -
      + {% if not column.sortable %} {{ column.name }} {% else %} @@ -31,7 +29,6 @@
      -{% endif %} -{% if not display_rows %} +{% else %}

      0 records

      {% endif %} diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..04181531 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -33,12 +33,9 @@ p.message-warning {

      Debug allow rules

      -{% set current_tab = "allow_debug" %} -{% include "_permissions_debug_tabs.html" %} -

      Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

      -
      +

      diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index dc393c20..109fb1e9 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -19,7 +19,7 @@

      GET - +
      @@ -29,7 +29,7 @@
      POST - +
      diff --git a/datasette/templates/base.html b/datasette/templates/base.html index e1767deb..83f87614 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,5 +1,5 @@ {% import "_crumbs.html" as crumbs with context %} - + {% block title %}{% endblock %} @@ -7,8 +7,6 @@ {% for url in extra_css_urls %} {% endfor %} - - {% for url in extra_js_urls %} {% endfor %} @@ -19,8 +17,8 @@
      -
      {% endif %} {% if actor %}
      {{ display_actor(actor) }} @@ -70,7 +70,5 @@ {% endfor %} {% if select_templates %}{% endif %} - - diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html index 270d9c86..2be98d38 100644 --- a/datasette/templates/create_token.html +++ b/datasette/templates/create_token.html @@ -39,7 +39,7 @@ {% endfor %} {% endif %} -
      +
      +
      Restrict actions that can be performed using this token

      All databases and tables

        - {% for permission in all_actions %} + {% for permission in all_permissions %}
        • {% endfor %}
      @@ -64,7 +65,7 @@ {% for database in database_with_tables %}

      All tables in "{{ database.name }}"

        - {% for permission in database_actions %} + {% for permission in database_permissions %}
        • {% endfor %}
      @@ -74,7 +75,7 @@ {% for table in database.tables %}

      {{ database.name }}: {{ table.name }}

        - {% for permission in child_actions %} + {% for permission in resource_permissions %}
        • {% endfor %}
      diff --git a/datasette/templates/csrf_error.html b/datasette/templates/csrf_error.html deleted file mode 100644 index b84749d3..00000000 --- a/datasette/templates/csrf_error.html +++ /dev/null @@ -1,13 +0,0 @@ -{% extends "base.html" %} -{% block title %}CSRF check failed{% endblock %} -{% block content %} -

      Form origin check failed

      - -

      Your request's origin could not be validated. Please return to the form and submit it again.

      - -
      Technical details -

      Developers: consult Datasette's CSRF protection documentation.

      -

      Reason: {{ reason }}

      -
      - -{% endblock %} diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 371f6a22..3d4dae07 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -5,34 +5,41 @@ {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} -{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} -{% block crumbs %} -{{ crumbs.nav(request=request, database=database) }} -{% endblock %} - {% block content %}

      {{ metadata.title or database }}{% if private %} 🔒{% endif %}

      -
      -{% set action_links, action_title = database_actions(), "Database actions" %} -{% include "_action_menu.html" %} - -{{ top_database() }} + {% set links = database_actions() %}{% if links %} +
      + + Table actions + + + +
      + {% if links %} + + {% endif %} +
      +
      {% endif %} +
      {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} - +

      Custom SQL query

      -

      - {% set parameter_names = [] %} - {% set parameter_values = {} %} - {% set sql_parameters_allow_expand = false %} - {% include "_sql_parameters.html" %} +

      @@ -45,7 +52,7 @@

      The following databases are attached to this connection, and can be used for cross-database joins:

        {% for db_name in attached_databases %} -
      • {{ db_name }} - tables
        • +
      • {{ db_name }} - tables
        • {% endfor %}
      @@ -58,13 +65,10 @@
    • {{ query.title or query.name }}{% if query.private %} 🔒{% endif %}
      • {% endfor %} - {% if queries_more %} -

      View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

      - {% endif %} {% endif %} {% if tables %} -

      Tables schema

      +

      Tables

      {% endif %} {% for table in tables %} @@ -72,7 +76,7 @@

      {{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

      {% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

      -

      {% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

      +

      {% if table.count is none %}Many rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

      {% endif %} {% endfor %} @@ -91,15 +95,9 @@ {% endif %} {% if allow_download %} -

      Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

      +

      Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

      {% endif %} {% include "_codemirror_foot.html" %} -{% include "_sql_parameter_scripts.html" %} - {% endblock %} diff --git a/datasette/templates/debug_actions.html b/datasette/templates/debug_actions.html deleted file mode 100644 index 0ef7b329..00000000 --- a/datasette/templates/debug_actions.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Registered Actions{% endblock %} - -{% block content %} -

      Registered actions

      - -{% set current_tab = "actions" %} -{% include "_permissions_debug_tabs.html" %} - -

      - This Datasette instance has registered {{ data|length }} action{{ data|length != 1 and "s" or "" }}. - Actions are used by the permission system to control access to different features. -

      - - - - - - - - - - - - - - - {% for action in data %} - - - - - - - - - - {% endfor %} - -
      NameAbbrDescriptionResourceTakes ParentTakes ChildAlso Requires
      {{ action.name }}{% if action.abbr %}{{ action.abbr }}{% endif %}{{ action.description or "" }}{% if action.resource_class %}{{ action.resource_class }}{% endif %}{% if action.takes_parent %}✓{% endif %}{% if action.takes_child %}✓{% endif %}{% if action.also_requires %}{{ action.also_requires }}{% endif %}
      - -{% endblock %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html deleted file mode 100644 index add3154a..00000000 --- a/datasette/templates/debug_allowed.html +++ /dev/null @@ -1,229 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Allowed Resources{% endblock %} - -{% block extra_head %} - -{% include "_permission_ui_styles.html" %} -{% include "_debug_common_functions.html" %} -{% endblock %} - -{% block content %} -

      Allowed resources

      - -{% set current_tab = "allowed" %} -{% include "_permissions_debug_tabs.html" %} - -

      Use this tool to check which resources the current actor is allowed to access for a given permission action. It queries the /-/allowed.json API endpoint.

      - -{% if request.actor %} -

      Current actor: {{ request.actor.get("id", "anonymous") }}

      -{% else %} -

      Current actor: anonymous (not logged in)

      -{% endif %} - -
      - -
      - - - Only certain actions are supported by this endpoint -
      - -
      - - - Filter results to a specific parent resource -
      - -
      - - - Filter results to a specific child resource (requires parent to be set) -
      - -
      - - - Number of results per page (max 200) -
      - -
      - -
      - -
      - - - - - -{% endblock %} diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html deleted file mode 100644 index c2e7997f..00000000 --- a/datasette/templates/debug_check.html +++ /dev/null @@ -1,270 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Permission Check{% endblock %} - -{% block extra_head %} - -{% include "_permission_ui_styles.html" %} -{% include "_debug_common_functions.html" %} - -{% endblock %} - -{% block content %} -

      Permission check

      - -{% set current_tab = "check" %} -{% include "_permissions_debug_tabs.html" %} - -

      Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

      - -{% if request.actor %} -

      Current actor: {{ request.actor.get("id", "anonymous") }}

      -{% else %} -

      Current actor: anonymous (not logged in)

      -{% endif %} - -
      -
      -
      - - - The permission action to check -
      - -
      - - - For database-level permissions, specify the database name -
      - -
      - - - For table-level permissions, specify the table name (requires parent) -
      - -
      - -
      -
      -
      - - - - - -{% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html deleted file mode 100644 index 4410a677..00000000 --- a/datasette/templates/debug_permissions_playground.html +++ /dev/null @@ -1,165 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Debug permissions{% endblock %} - -{% block extra_head %} -{% include "_permission_ui_styles.html" %} - -{% endblock %} - -{% block content %} -

      Permission playground

      - -{% set current_tab = "permissions" %} -{% include "_permissions_debug_tabs.html" %} - -

      This tool lets you simulate an actor and a permission check for that actor.

      - -
      -
      -
      -
      - - -
      -
      -
      -
      - - -
      -
      - - -
      -
      - - -
      -
      -
      - -
      - 
      
-    
      
-
      
-
-
-
-
      Recent permissions checks
      
-
-
      
-    {% if filter != "all" %}All{% else %}All{% endif %},
-    {% if filter != "exclude-yours" %}Exclude yours{% else %}Exclude yours{% endif %},
-    {% if filter != "only-yours" %}Only yours{% else %}Only yours{% endif %}
-
      
-
-{% if permission_checks %}
-
-    
-        
-            
-            
-            
-            
-            
-            
-        
-    
-    
-        {% for check in permission_checks %}
-            
-                
-                
-                
-                
-                
-                
-            
-        {% endfor %}
-        
-
      WhenActionParentChildActorResult
      {{ check.when.split('T', 1)[0] }}
      {{ check.when.split('T', 1)[1].split('+', 1)[0].split('-', 1)[0].split('Z', 1)[0] }}      		{{ check.action }}{{ check.parent or '—' }}{{ check.child or '—' }}{% if check.actor %}{{ check.actor|tojson }}{% else %}anonymous{% endif %}{% if check.result %}Allowed{% elif check.result is none %}No opinion{% else %}Denied{% endif %}
      
-{% else %}
-
      No permission checks have been recorded yet.
      
-{% endif %}
-
-{% endblock %}
diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html
deleted file mode 100644
index 9a290803..00000000
--- a/datasette/templates/debug_rules.html
+++ /dev/null
@@ -1,203 +0,0 @@
-{% extends "base.html" %}
-
-{% block title %}Permission Rules{% endblock %}
-
-{% block extra_head %}
-
-{% include "_permission_ui_styles.html" %}
-{% include "_debug_common_functions.html" %}
-{% endblock %}
-
-{% block content %}
-
      Permission rules
      
-
-{% set current_tab = "rules" %}
-{% include "_permissions_debug_tabs.html" %}
-
-
      Use this tool to view the permission rules that allow the current actor to access resources for a given permission action. It queries the /-/rules.json API endpoint.
      
-
-{% if request.actor %}
-
      Current actor: {{ request.actor.get("id", "anonymous") }}
      
-{% else %}
-
      Current actor: anonymous (not logged in)
      
-{% endif %}
-
-
      
-    
      
-        
      
-            
-            
-            The permission action to check
-        
      
-
-        
      
-            
-            
-            Number of results per page (max 200)
-        
      
-
-        
      
-            
-        
      
-    
      
-
      
-
-
-
-
-
-{% endblock %}
diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html
deleted file mode 100644
index 6b626f8d..00000000
--- a/datasette/templates/execute_write.html
+++ /dev/null
@@ -1,314 +0,0 @@
-{% extends "base.html" %}
-
-{% block title %}Write to this database{% endblock %}
-
-{% block extra_head %}
-{{- super() -}}
-{% include "_codemirror.html" %}
-
-{% include "_execute_write_analysis_styles.html" %}
-{% include "_sql_parameter_styles.html" %}
-{% endblock %}
-
-{% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %}
-
-{% block crumbs %}
-{{ crumbs.nav(request=request, database=database) }}
-{% endblock %}
-
-{% block content %}
-
-
      Write to this database
      
-
-
      Execute SQL to insert, update or delete rows in this database.
      
-
-{% if execution_message %}
-    
      {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}
      
-{% endif %}
-
-
      
-    {% if write_template_tables %}
-        
      
-            
      
-                Start with a template
-                
      
-                    
-                    
-                    
-                    
-                    
-                
      
-            
      
-        
      
-    {% endif %}
-
-    
      
-
-    {% set sql_parameters_section_id = "execute-write-parameters-section" %}
-    {% set sql_parameters_allow_expand = true %}
-    {% include "_sql_parameters.html" %}
-
-    
      
-        
      Query operations
      
-        {% if analysis_error %}
-            
      {{ analysis_error }}
      
-        {% elif analysis_rows %}
-            
      
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                
-                
-                    {% for row in analysis_rows %}
-                        
-                            
-                            
-                            
-                            
-                            
-                        
-                    {% endfor %}
-                
-            
      OperationDatabaseTableRequired permissionAllowed
      {{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
      
-        {% else %}
-            
      Analysis will show each affected table and required permission.
      
-        {% endif %}
-    
      
-
-    
      
-        
-        {% if save_query_base_url %}Save this query{% endif %}
-    
      
-
      
-
-
-
-{% include "_codemirror_foot.html" %}
-{% include "_sql_parameter_scripts.html" %}
-{% include "_execute_write_analysis_scripts.html" %}
-
-
-
-{% if write_template_tables %}
-
-{% endif %}
-
-{% endblock %}
diff --git a/datasette/templates/index.html b/datasette/templates/index.html
index 03349279..06e09635 100644
--- a/datasette/templates/index.html
+++ b/datasette/templates/index.html
@@ -2,26 +2,17 @@
 
 {% block title %}{{ metadata.title or "Datasette" }}: {% for database in databases %}{{ database.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% endblock %}
 
-{% block extra_head %}
-{% if noindex %}{% endif %}
-{% endblock %}
-
 {% block body_class %}index{% endblock %}
 
 {% block content %}
 
      {{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}
      
 
-{% set action_links, action_title = homepage_actions, "Homepage actions" %}
-{% include "_action_menu.html" %}
-
-{{ top_homepage() }}
-
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
 {% for database in databases %}
     
      {{ database.name }}{% if database.private %} 🔒{% endif %}
      
     
      
-        {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.hidden_tables_count %}, {% endif -%}
+        {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%}
         {% if database.hidden_tables_count -%}
             {% if database.show_table_row_counts %}{{ "{:,}".format(database.hidden_table_rows_sum) }} rows in {% endif %}{{ database.hidden_tables_count }} hidden table{% if database.hidden_tables_count != 1 %}s{% endif -%}
         {% endif -%}
diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html
index a99870e6..4c4a7d11 100644
--- a/datasette/templates/logout.html
+++ b/datasette/templates/logout.html
@@ -8,8 +8,9 @@
 
 
      You are logged in as {{ display_actor(actor) }}
      
 
-
      
+
     
      
+        
         
     
      
 
      
diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html
index 891cf915..e0ab9a40 100644
--- a/datasette/templates/messages_debug.html
+++ b/datasette/templates/messages_debug.html
@@ -8,7 +8,7 @@
 
 
      Set a message:
      
 
-
      
+
     
      
         
         
      
@@ -19,6 +19,7 @@
                 
             
         
      
+        
         
     
      
 
      
diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html
index 7770f7d4..9905df2c 100644
--- a/datasette/templates/patterns.html
+++ b/datasette/templates/patterns.html
@@ -1,5 +1,5 @@
 
-
+
 
     Datasette: Pattern Portfolio
     
@@ -9,7 +9,7 @@
 
 
 
-
      
@@ -45,7 +45,7 @@
 
 
      Header for /database/table/row and Messages
      
 
-
      
+