diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml deleted file mode 100644 index e56d9c27..00000000 --- a/.github/workflows/deploy-branch-preview.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy a Datasette branch preview to Vercel - -on: - workflow_dispatch: - inputs: - branch: - description: "Branch to deploy" - required: true - type: string - -jobs: - deploy-branch-preview: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install datasette-publish-vercel - - name: Deploy the preview - env: - VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} - run: | - export BRANCH="${{ github.event.inputs.branch }}" - wget https://latest.datasette.io/fixtures.db - datasette publish vercel fixtures.db \ - --branch $BRANCH \ - --project "datasette-preview-$BRANCH" \ - --token $VERCEL_TOKEN \ - --scope datasette \ - --about "Preview of $BRANCH" \ - --about_url "https://github.com/simonw/datasette/tree/$BRANCH" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b6c58c7e..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -1,10 +1,11 @@ name: Deploy latest.datasette.io on: + workflow_dispatch: push: branches: - main - - cloudrun-fix + # - 1.0-dev permissions: contents: read @@ -14,24 +15,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 - # Using Python 3.10 for gcloud compatibility: with: - python-version: "3.10" - - uses: actions/cache@v4 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.13" + cache: pip - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] - python -m pip install -e .[docs] + python -m pip install . --group dev python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests if: ${{ github.ref == 'refs/heads/main' }} @@ -64,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index a54bd83a..b8fb8aaa 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request_target: + pull_request: types: - opened diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 77cce7d1..735e14e9 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 - - uses: actions/cache@v4 + uses: actions/checkout@v6 + - uses: actions/cache@v5 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6c83346d..87300593 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,16 +14,16 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev - name: Run tests run: | pytest @@ -35,13 +35,13 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.13' cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install setuptools wheel build @@ -56,16 +56,16 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.10' cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - python -m pip install -e .[docs] + python -m pip install . --group dev python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index afe8d6b2..e622ef4c 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 8a47fd2d..9a808194 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,16 +9,16 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.11' cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: '**/pyproject.toml' - name: Install dependencies run: | - pip install -e '.[docs]' + pip install . --group dev - name: Check spelling run: | codespell README.md --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml new file mode 100644 index 00000000..59b5fbc0 --- /dev/null +++ b/.github/workflows/stable-docs.yml @@ -0,0 +1,76 @@ +name: Update Stable Docs + +on: + release: + types: [published] + push: + branches: + - main + +permissions: + contents: write + +jobs: + update_stable_docs: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 # We need all commits to find docs/ changes + - name: Set up Git user + run: | + git config user.name "Automated" + git config user.email "actions@users.noreply.github.com" + - name: Create stable branch if it does not yet exist + run: | + if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then + # Make sure we have all tags locally + git fetch --tags --quiet + + # Latest tag that is just numbers and dots (optionally prefixed with 'v') + # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. + LATEST_RELEASE=$( + git tag -l --sort=-v:refname \ + | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ + | head -n1 + ) + + git checkout -b stable + + # If there are any stable releases, copy docs/ from the most recent + if [ -n "$LATEST_RELEASE" ]; then + rm -rf docs/ + git checkout "$LATEST_RELEASE" -- docs/ || true + fi + + git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" + git push -u origin stable + fi + - name: Handle Release + if: github.event_name == 'release' && !github.event.release.prerelease + run: | + git fetch --all + git checkout stable + git reset --hard ${GITHUB_REF#refs/tags/} + git push origin stable --force + - name: Handle Commit to Main + if: contains(github.event.head_commit.message, '!stable-docs') + run: | + git fetch origin + git checkout -b stable origin/stable + # Get the list of modified files in docs/ from the current commit + FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) + # Check if the list of files is non-empty + if [[ -n "$FILES" ]]; then + # Checkout those files to the stable branch to over-write with their contents + for FILE in $FILES; do + git checkout ${{ github.sha }} -- $FILE + done + git add docs/ + git commit -m "Doc changes from ${{ github.sha }}" + git push origin stable + else + echo "No changes to docs/ in this commit." + exit 0 + fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 22a69150..c514048e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,17 +15,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.12' cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: '**/pyproject.toml' - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] + python -m pip install . --group dev python -m pip install pytest-cov - name: Run tests run: |- diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 7357b30c..5162c47a 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,15 +12,15 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: python-version: "3.10" cache: 'pip' - cache-dependency-path: '**/setup.py' + cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 698aec8a..23fce459 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,14 +25,14 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 with: @@ -45,7 +45,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev pip freeze - name: Run tests run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 901c4905..a1b2e9d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,20 +12,20 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev pip freeze - name: Run tests run: | @@ -33,9 +33,12 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Install docs dependencies + - name: Black run: | - pip install -e '.[docs]' + black --version + black --check . + - name: Ruff + run: ruff check datasette tests - name: Check if cog needs to be run run: | cog --check docs/*.rst diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index fcee0f21..a033cd92 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 9792245d..72af1eec 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -5,11 +5,14 @@ on: permissions: contents: read + models: read jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 277ff653..12acd87e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,12 @@ scratchpad .vscode +uv.lock +data.db + +# test databases +*.db + # We don't use Pipfile, so ignore them Pipfile Pipfile.lock @@ -123,4 +129,6 @@ node_modules # include it in source control. tests/*.dylib tests/*.so -tests/*.dll \ No newline at end of file +tests/*.dll + +.idea \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 5b30e75a..8b3e54aa 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,16 +1,17 @@ version: 2 -build: - os: ubuntu-20.04 - tools: - python: "3.11" - sphinx: - configuration: docs/conf.py + configuration: docs/conf.py -python: - install: - - method: pip - path: . - extra_requirements: - - docs +build: + os: ubuntu-24.04 + tools: + python: "3.13" + jobs: + install: + - pip install --upgrade pip + - pip install . --group dev + +formats: +- pdf +- epub diff --git a/Dockerfile b/Dockerfile index e7497690..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.14-slim-trixie as build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/Justfile b/Justfile index 172de444..657881be 100644 --- a/Justfile +++ b/Justfile @@ -5,38 +5,56 @@ export DATASETTE_SECRET := "not_a_secret" # Setup project @init: - pipenv run pip install -e '.[test,docs]' + uv sync # Run pytest with supplied options -@test *options: - pipenv run pytest {{options}} +@test *options: init + uv run pytest -n auto {{options}} @codespell: - pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt - pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - pipenv run codespell tests --ignore-words docs/codespell-ignore-words.txt + uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + uv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, flake8, mypy, cog +# Run linters: black, ruff, cog @lint: codespell - pipenv run black . --check - pipenv run flake8 - pipenv run cog --check README.md docs/*.rst + uv run black datasette tests --check + uv run ruff check datasette tests + uv run cog --check README.md docs/*.rst + +# Apply ruff fixes +@fix: + uv run ruff check --fix datasette tests # Rebuild docs with cog @cog: - pipenv run cog -r README.md docs/*.rst + uv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: cog - pipenv run blacken-docs -l 60 docs/*.rst - cd docs && pipenv run make livehtml +@docs: cog blacken-docs + uv run make -C docs livehtml + +# Build docs as static HTML +@docs-build: cog blacken-docs + rm -rf docs/_build && cd docs && uv run make html # Apply Black @black: - pipenv run black . + uv run black datasette tests -@serve: - pipenv run sqlite-utils create-database data.db - pipenv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore - pipenv run python -m datasette data.db --root --reload +# Apply blacken-docs +@blacken-docs: + uv run blacken-docs -l 60 docs/*.rst + +# Apply prettier +@prettier: + npm run fix + +# Format code with both black and prettier +@format: black prettier blacken-docs + +@serve *options: + uv run sqlite-utils create-database data.db + uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + uv run python -m datasette data.db --root --reload {{options}} diff --git a/datasette/__init__.py b/datasette/__init__.py index 47d2b4f6..eb18e59e 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,6 +1,7 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa +from datasette.tokens import TokenHandler, TokenRestrictions # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py new file mode 100644 index 00000000..5fb6b473 --- /dev/null +++ b/datasette/_pytest_plugin.py @@ -0,0 +1,108 @@ +""" +Pytest plugin that automatically closes any Datasette instances constructed +during a pytest test — both in the test body and in function-scoped +fixtures. Instances constructed by session-, module-, class- or package- +scoped fixtures are left alone, because other tests in the session will +still want to use them. + +Registered as a pytest11 entry point in pyproject.toml so that downstream +projects using Datasette get the same FD-safety net for their own tests. + +Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the +equivalent ini file). +""" + +from __future__ import annotations + +import contextvars +import weakref + +import pytest + +from datasette.app import Datasette + +_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "datasette_active_instances", default=None +) + +_original_init = Datasette.__init__ + + +def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + +Datasette.__init__ = _tracking_init + + +def pytest_addoption(parser): + parser.addini( + "datasette_autoclose", + help=( + "Automatically close Datasette instances created inside test " + "bodies and function-scoped fixtures (default: true)." + ), + default="true", + ) + + +def _enabled(config) -> bool: + value = config.getini("datasette_autoclose") + if isinstance(value, bool): + return value + return str(value).strip().lower() not in ("false", "0", "no", "off") + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + """Track Datasette instances across setup, call and teardown; close at end.""" + if not _enabled(item.config): + yield + return + refs: list[weakref.ref] = [] + token = _active_instances.set(refs) + try: + yield + finally: + _active_instances.reset(token) + for ref in reversed(refs): + ds = ref() + if ds is None: + continue + try: + ds.close() + except Exception as e: + item.warn( + pytest.PytestUnraisableExceptionWarning( + f"Error closing Datasette instance: {e!r}" + ) + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request): + """Exempt instances created by non-function-scoped fixtures. + + Session-, module-, class- and package-scoped fixtures produce Datasette + instances that must survive beyond the current test — other tests in + the session will still use them. When such a fixture creates one or + more Datasette instances during its setup, we snapshot the tracking + list before the fixture runs and subtract off any instances that were + added during its setup, so they don't get closed at test teardown. + """ + refs = _active_instances.get() + if refs is None: + yield + return + before_ids = {id(ref) for ref in refs} + yield + if fixturedef.scope != "function": + new_refs = [ref for ref in refs if id(ref) not in before_ids] + for new_ref in new_refs: + try: + refs.remove(new_ref) + except ValueError: + pass diff --git a/datasette/app.py b/datasette/app.py index 6c7026a8..e7f34e69 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,7 +1,12 @@ -from asgi_csrf import Errors +from __future__ import annotations + import asyncio -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union -import asgi_csrf +import contextvars +from typing import TYPE_CHECKING, Any, Dict, Iterable, List + +if TYPE_CHECKING: + from datasette.permissions import Resource + from datasette.tokens import TokenRestrictions import collections import dataclasses import datetime @@ -36,8 +41,26 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .events import Event +from .column_types import SQLiteType +from . import stored_queries, write_sql from .views import Context -from .views.database import database_download, DatabaseView, TableCreateView, QueryView +from .views.database import ( + database_download, + DatabaseView, + TableCreateView, + QueryView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) from .views.index import IndexView from .views.special import ( JsonDataView, @@ -52,10 +75,15 @@ from .views.special import ( AllowedResourcesView, PermissionRulesView, PermissionCheckView, + JumpView, + InstanceSchemaView, + DatabaseSchemaView, + TableSchemaView, ) from .views.table import ( TableInsertView, TableUpsertView, + TableSetColumnTypeView, TableDropView, table_view, ) @@ -65,6 +93,7 @@ from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( + PaginatedResources, PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, @@ -85,6 +114,7 @@ from .utils import ( resolve_env_secrets, resolve_routes, tilde_decode, + tilde_encode, to_css_class, urlsafe_components, redact_keys, @@ -105,6 +135,7 @@ from .utils.asgi import ( asgi_send_file, asgi_send_redirect, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -114,10 +145,39 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .utils.permissions import build_rules_union, PluginSQL +from .resources import DatabaseResource, TableResource app_root = Path(__file__).parent.parent + +# Context variable to track when code is executing within a datasette.client request +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + + +class _DatasetteClientContext: + """Context manager to mark code as executing within a datasette.client request.""" + + def __enter__(self): + self.token = _in_datasette_client.set(True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + _in_datasette_client.reset(self.token) + return False + + +@dataclasses.dataclass +class PermissionCheck: + """Represents a logged permission check for debugging purposes.""" + + when: str + actor: Dict[str, Any] | None + action: str + parent: str | None + child: str | None + result: bool + + # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 SQLITE_LIMIT_ATTACHED = 10 @@ -227,6 +287,9 @@ FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() +ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) + + async def favicon(request, send): await asgi_send_file( send, @@ -277,8 +340,10 @@ class Datasette: crossdb=False, nolock=False, internal=None, + default_deny=False, ): self._startup_invoked = False + self._closed = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -307,7 +372,8 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() - self.permissions = {} # .invoke_startup() will populate this + self.actions = {} # .invoke_startup() will populate this + self._column_types = {} # .invoke_startup() will populate this try: self._refresh_schemas_lock = asyncio.Lock() except RuntimeError as rex: @@ -332,7 +398,7 @@ class Datasette: self.internal_db_created = False if internal is None: - self._internal_database = Database(self, memory_name=secrets.token_hex()) + self._internal_database = Database(self, is_temp_disk=True) else: self._internal_database = Database(self, path=internal, mode="rwc") self._internal_database.name = INTERNAL_DB_NAME @@ -391,10 +457,37 @@ class Datasette: config = config or {} config_settings = config.get("settings") or {} - # validate "settings" keys in datasette.json - for key in config_settings: + # Validate settings from config file + for key, value in config_settings.items(): if key not in DEFAULT_SETTINGS: - raise StartupError("Invalid setting '{}' in datasette.json".format(key)) + raise StartupError(f"Invalid setting '{key}' in config file") + # Validate type matches expected type from DEFAULT_SETTINGS + if value is not None: # Allow None/null values + expected_type = type(DEFAULT_SETTINGS[key]) + actual_type = type(value) + if actual_type != expected_type: + raise StartupError( + f"Setting '{key}' in config file has incorrect type. " + f"Expected {expected_type.__name__}, got {actual_type.__name__}. " + f"Value: {value!r}. " + f"Hint: In YAML/JSON config files, remove quotes from boolean and integer values." + ) + + # Validate settings from constructor parameter + if settings: + for key, value in settings.items(): + if key not in DEFAULT_SETTINGS: + raise StartupError(f"Invalid setting '{key}' in settings parameter") + if value is not None: + expected_type = type(DEFAULT_SETTINGS[key]) + actual_type = type(value) + if actual_type != expected_type: + raise StartupError( + f"Setting '{key}' in settings parameter has incorrect type. " + f"Expected {expected_type.__name__}, got {actual_type.__name__}. " + f"Value: {value!r}" + ) + self.config = config # CLI settings should overwrite datasette.json settings self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) @@ -457,6 +550,8 @@ class Datasette: self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) + self.root_enabled = False + self.default_deny = default_deny self.client = DatasetteClient(self) async def apply_metadata_json(self): @@ -493,6 +588,9 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file + async def _save_queries_from_config(self): + await stored_queries.save_queries_from_config(self) + def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: @@ -502,21 +600,23 @@ class Datasette: pass return environment - def get_permission(self, name_or_abbr: str) -> "Permission": + def get_action(self, name_or_abbr: str): """ - Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. + Returns an Action object for the given name or abbreviation. Returns None if not found. """ - if name_or_abbr in self.permissions: - return self.permissions[name_or_abbr] + if name_or_abbr in self.actions: + return self.actions[name_or_abbr] # Try abbreviation - for permission in self.permissions.values(): - if permission.abbr == name_or_abbr: - return permission - raise KeyError( - "No permission found with name or abbreviation {}".format(name_or_abbr) - ) + for action in self.actions.values(): + if action.abbr == name_or_abbr: + return action + return None async def refresh_schemas(self): + # Throttle schema refreshes to at most once per second + if time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0: + return + self._last_schema_refresh = time.monotonic() if self._refresh_schemas_lock.locked(): return async with self._refresh_schemas_lock: @@ -534,6 +634,36 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } + catalog_table_names = ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", + ) + # Delete stale entries for databases that are no longer attached + catalog_database_names = set(current_schema_versions.keys()) + for table in catalog_table_names[:-1]: + catalog_database_names.update( + row["database_name"] + for row in await internal_db.execute( + "select distinct database_name from {}".format(table) + ) + if row["database_name"] is not None + ) + stale_databases = catalog_database_names - set(self.databases.keys()) + if stale_databases: + + def delete_stale_database_catalog(conn): + for stale_db_name in stale_databases: + for table in catalog_table_names: + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [stale_db_name], + ) + + await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it @@ -548,9 +678,7 @@ class Datasette: """ INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) VALUES {} - """.format( - placeholders - ), + """.format(placeholders), values, ) await populate_schema_tables(internal_db, db) @@ -559,6 +687,17 @@ class Datasette: def urls(self): return Urls(self) + @property + def pm(self): + """ + Return the global plugin manager instance. + + This provides access to the pluggy PluginManager that manages all + Datasette plugins and hooks. Use datasette.pm.hook.hook_name() to + call plugin hooks. + """ + return pm + async def invoke_startup(self): # This must be called for Datasette to be in a usable state if self._startup_invoked: @@ -571,28 +710,50 @@ class Datasette: event_classes.extend(extra_classes) self.event_classes = tuple(event_classes) - # Register permissions, but watch out for duplicate name/abbr - names = {} - abbrs = {} - for hook in pm.hook.register_permissions(datasette=self): + # Register actions, but watch out for duplicate name/abbr + action_names = {} + action_abbrs = {} + for hook in pm.hook.register_actions(datasette=self): if hook: - for p in hook: - if p.name in names and p != names[p.name]: + for action in hook: + if ( + action.name in action_names + and action != action_names[action.name] + ): raise StartupError( - "Duplicate permission name: {}".format(p.name) + "Duplicate action name: {}".format(action.name) ) - if p.abbr and p.abbr in abbrs and p != abbrs[p.abbr]: + if ( + action.abbr + and action.abbr in action_abbrs + and action != action_abbrs[action.abbr] + ): raise StartupError( - "Duplicate permission abbr: {}".format(p.abbr) + "Duplicate action abbr: {}".format(action.abbr) ) - names[p.name] = p - if p.abbr: - abbrs[p.abbr] = p - self.permissions[p.name] = p + action_names[action.name] = action + if action.abbr: + action_abbrs[action.abbr] = action + self.actions[action.name] = action + + # Register column types (classes, not instances) + self._column_types = {} + for hook in pm.hook.register_column_types(datasette=self): + if hook: + for ct_cls in hook: + if ct_cls.name in self._column_types: + raise StartupError(f"Duplicate column type name: {ct_cls.name}") + self._column_types[ct_cls.name] = ct_cls + for hook in pm.hook.prepare_jinja2_environment( env=self._jinja_env, datasette=self ): await await_me_maybe(hook) + # Ensure internal tables and metadata are populated before startup hooks + await self._refresh_schemas() + await self._save_queries_from_config() + # Load column_types from config into internal DB + await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) self._startup_invoked = True @@ -603,44 +764,78 @@ class Datasette: def unsign(self, signed, namespace="default"): return URLSafeSerializer(self._secret, namespace).loads(signed) - def create_token( + def in_client(self) -> bool: + """Check if the current code is executing within a datasette.client request. + + Returns: + bool: True if currently executing within a datasette.client request, False otherwise. + """ + return _in_datasette_client.get() + + def _token_handlers(self): + """Collect all registered token handlers from plugins.""" + from datasette.tokens import TokenHandler + + handlers = [] + for result in pm.hook.register_token_handler(datasette=self): + if isinstance(result, TokenHandler): + handlers.append(result) + elif isinstance(result, list): + handlers.extend(h for h in result if isinstance(h, TokenHandler)) + return handlers + + async def create_token( self, actor_id: str, *, - expires_after: Optional[int] = None, - restrict_all: Optional[Iterable[str]] = None, - restrict_database: Optional[Dict[str, Iterable[str]]] = None, - restrict_resource: Optional[Dict[str, Dict[str, Iterable[str]]]] = None, - ): - token = {"a": actor_id, "t": int(time.time())} - if expires_after: - token["d"] = expires_after + expires_after: int | None = None, + restrictions: "TokenRestrictions | None" = None, + handler: str | None = None, + ) -> str: + """ + Create an API token for the given actor. - def abbreviate_action(action): - # rename to abbr if possible - permission = self.permissions.get(action) - if not permission: - return action - return permission.abbr or action + Uses the first registered token handler by default, or a specific + handler if ``handler`` is provided (matched by handler name). - if expires_after: - token["d"] = expires_after - if restrict_all or restrict_database or restrict_resource: - token["_r"] = {} - if restrict_all: - token["_r"]["a"] = [abbreviate_action(a) for a in restrict_all] - if restrict_database: - token["_r"]["d"] = {} - for database, actions in restrict_database.items(): - token["_r"]["d"][database] = [abbreviate_action(a) for a in actions] - if restrict_resource: - token["_r"]["r"] = {} - for database, resources in restrict_resource.items(): - for resource, actions in resources.items(): - token["_r"]["r"].setdefault(database, {})[resource] = [ - abbreviate_action(a) for a in actions - ] - return "dstok_{}".format(self.sign(token, namespace="token")) + Pass a :class:`TokenRestrictions` to limit which actions the token + can perform. + """ + handlers = self._token_handlers() + if not handlers: + raise RuntimeError("No token handlers are registered") + + if handler is not None: + matched = [h for h in handlers if h.name == handler] + if not matched: + available = [h.name for h in handlers] + raise ValueError( + f"Token handler {handler!r} not found. " + f"Available handlers: {available}" + ) + chosen = matched[0] + else: + chosen = handlers[0] + + return await chosen.create_token( + self, + actor_id, + expires_after=expires_after, + restrictions=restrictions, + ) + + async def verify_token(self, token: str) -> dict | None: + """ + Verify an API token by trying all registered token handlers. + + Returns an actor dict from the first handler that recognizes the + token, or None if no handler accepts it. + """ + for token_handler in self._token_handlers(): + result = await token_handler.verify_token(self, token) + if result is not None: + return result + return None def get_database(self, name=None, route=None): if route is not None: @@ -671,8 +866,10 @@ class Datasette: self.databases = new_databases return db - def add_memory_database(self, memory_name): - return self.add_database(Database(self, memory_name=memory_name)) + def add_memory_database(self, memory_name, name=None, route=None): + return self.add_database( + Database(self, memory_name=memory_name), name=name, route=route + ) def remove_database(self, name): self.get_database(name).close() @@ -680,6 +877,33 @@ class Datasette: new_databases.pop(name) self.databases = new_databases + def close(self): + """Release all resources held by this Datasette instance. + + Closes every attached Database (including the internal database), + shuts down the executor, and unlinks the temporary file used for + the internal database if one was created. Idempotent and one-way. + """ + if self._closed: + return + self._closed = True + first_exception = None + dbs = list(self.databases.values()) + [self._internal_database] + for db in dbs: + try: + db.close() + except Exception as e: + if first_exception is None: + first_exception = e + if self.executor is not None: + try: + self.executor.shutdown(wait=True, cancel_futures=True) + except Exception as e: + if first_exception is None: + first_exception = e + if first_exception is not None: + raise first_exception + def setting(self, key): return self._settings.get(key, None) @@ -699,14 +923,12 @@ class Datasette: return orig async def get_instance_metadata(self): - rows = await self.get_internal_database().execute( - """ + rows = await self.get_internal_database().execute(""" SELECT key, value FROM metadata_instance - """ - ) + """) return dict(rows) async def get_database_metadata(self, database_name: str): @@ -806,6 +1028,349 @@ class Datasette: [database_name, resource_name, column_name, key, value], ) + @staticmethod + def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: + return stored_queries.query_row_to_stored_query(row) + + @staticmethod + def _query_options_json(options): + return stored_queries.query_options_json(options) + + async def add_query( + self, + database: str, + name: str, + sql: str, + *, + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, + ) -> None: + return await stored_queries.add_query( + self, + database, + name, + sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + replace=replace, + ) + + async def update_query( + self, + database: str, + name: str, + *, + sql=stored_queries.UNCHANGED, + title=stored_queries.UNCHANGED, + description=stored_queries.UNCHANGED, + description_html=stored_queries.UNCHANGED, + hide_sql=stored_queries.UNCHANGED, + fragment=stored_queries.UNCHANGED, + parameters=stored_queries.UNCHANGED, + is_write=stored_queries.UNCHANGED, + is_private=stored_queries.UNCHANGED, + is_trusted=stored_queries.UNCHANGED, + source=stored_queries.UNCHANGED, + owner_id=stored_queries.UNCHANGED, + on_success_message=stored_queries.UNCHANGED, + on_success_message_sql=stored_queries.UNCHANGED, + on_success_redirect=stored_queries.UNCHANGED, + on_error_message=stored_queries.UNCHANGED, + on_error_redirect=stored_queries.UNCHANGED, + ) -> None: + return await stored_queries.update_query( + self, + database, + name, + sql=sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + ) + + async def remove_query( + self, database: str, name: str, source: str | None = None + ) -> None: + return await stored_queries.remove_query(self, database, name, source=source) + + async def get_query( + self, database: str, name: str + ) -> stored_queries.StoredQuery | None: + return await stored_queries.get_query(self, database, name) + + async def count_queries( + self, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + ) -> int: + return await stored_queries.count_queries( + self, + database, + actor=actor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + ) + + async def list_queries( + self, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, + ) -> stored_queries.StoredQueryPage: + return await stored_queries.list_queries( + self, + database, + actor=actor, + limit=limit, + cursor=cursor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + include_private=include_private, + ) + + async def ensure_query_write_permissions( + self, database, sql, *, actor=None, params=None, analysis=None + ): + # Raise Forbidden or QueryWriteRejected if SQL should not run + return await write_sql.ensure_query_write_permissions( + self, database, sql, actor=actor, params=params, analysis=analysis + ) + + # Column types API + + async def _get_resource_column_details(self, database: str, resource: str): + db = self.databases.get(database) + if db is None: + return {} + try: + return { + column.name: column + for column in await db.table_column_details(resource) + } + except sqlite3.OperationalError: + return {} + + @staticmethod + def _column_type_is_applicable(ct_cls, column_detail) -> bool: + sqlite_types = getattr(ct_cls, "sqlite_types", None) + if sqlite_types is None: + return True + if column_detail is None: + return False + actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type) + return actual_sqlite_type in sqlite_types + + async def _validate_column_type_assignment( + self, database: str, resource: str, column: str, ct_cls + ) -> None: + sqlite_types = getattr(ct_cls, "sqlite_types", None) + if sqlite_types is None: + return + + column_detail = ( + await self._get_resource_column_details(database, resource) + ).get(column) + if column_detail is None: + return + + actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type) + if actual_sqlite_type in sqlite_types: + return + + allowed = ", ".join(sqlite_type.value for sqlite_type in sqlite_types) + actual = ( + actual_sqlite_type.value + if actual_sqlite_type is not None + else "unrecognized {!r}".format(column_detail.type) + ) + raise ValueError( + "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " + "has SQLite type {}".format( + ct_cls.name, + allowed, + database, + resource, + column, + actual, + ) + ) + + async def _apply_column_types_config(self): + """Load column_types from datasette.json config into the internal DB.""" + import logging + + for db_name, db_conf in (self.config or {}).get("databases", {}).items(): + for table_name, table_conf in db_conf.get("tables", {}).items(): + for col_name, ct in table_conf.get("column_types", {}).items(): + if isinstance(ct, str): + col_type, config = ct, None + else: + col_type = ct["type"] + config = ct.get("config") + if col_type not in self._column_types: + logging.warning( + "column_types config references unknown type %r " + "for %s.%s.%s", + col_type, + db_name, + table_name, + col_name, + ) + try: + await self.set_column_type( + db_name, table_name, col_name, col_type, config + ) + except ValueError as ex: + logging.warning(str(ex)) + + async def get_column_type(self, database: str, resource: str, column: str): + """ + Return a ColumnType instance (with config baked in) for a specific + column, or None if no column type is assigned. + """ + row = await self.get_internal_database().execute( + "SELECT column_type, config FROM column_types " + "WHERE database_name = ? AND resource_name = ? AND column_name = ?", + [database, resource, column], + ) + rows = row.rows + if not rows: + return None + ct_name, config = rows[0] + ct_cls = self._column_types.get(ct_name) + if ct_cls is None: + return None + column_detail = ( + await self._get_resource_column_details(database, resource) + ).get(column) + if not self._column_type_is_applicable(ct_cls, column_detail): + return None + return ct_cls(config=json.loads(config) if config else None) + + async def get_column_types(self, database: str, resource: str) -> dict: + """ + Return {column_name: ColumnType instance (with config)} + for all columns with assigned types on the given resource. + """ + rows = await self.get_internal_database().execute( + "SELECT column_name, column_type, config FROM column_types " + "WHERE database_name = ? AND resource_name = ?", + [database, resource], + ) + column_details = await self._get_resource_column_details(database, resource) + result = {} + for row in rows.rows: + col_name, ct_name, config = row + ct_cls = self._column_types.get(ct_name) + if ct_cls is not None and self._column_type_is_applicable( + ct_cls, column_details.get(col_name) + ): + result[col_name] = ct_cls(config=json.loads(config) if config else None) + return result + + async def set_column_type( + self, + database: str, + resource: str, + column: str, + column_type: str, + config: dict = None, + ) -> None: + """Assign a column type. Overwrites any existing assignment.""" + ct_cls = self._column_types.get(column_type) + if ct_cls is not None: + await self._validate_column_type_assignment( + database, resource, column, ct_cls + ) + await self.get_internal_database().execute_write( + """INSERT OR REPLACE INTO column_types + (database_name, resource_name, column_name, column_type, config) + VALUES (?, ?, ?, ?, ?)""", + [ + database, + resource, + column, + column_type, + json.dumps(config) if config else None, + ], + ) + + async def remove_column_type( + self, database: str, resource: str, column: str + ) -> None: + """Remove a column type assignment.""" + await self.get_internal_database().execute_write( + "DELETE FROM column_types " + "WHERE database_name = ? AND resource_name = ? AND column_name = ?", + [database, resource, column], + ) + def get_internal_database(self): return self._internal_database @@ -849,38 +1414,24 @@ class Datasette: return db_plugin_config + def static_hash(self, filename): + if not hasattr(self, "_static_hashes"): + self._static_hashes = {} + path = os.path.join(str(app_root), "datasette/static", filename) + signature = (os.path.getmtime(path), os.path.getsize(path)) + cached = self._static_hashes.get(filename) + if cached and cached["signature"] == signature: + return cached["hash"] + with open(path) as fp: + static_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[:6] + self._static_hashes[filename] = { + "signature": signature, + "hash": static_hash, + } + return static_hash + def app_css_hash(self): - if not hasattr(self, "_app_css_hash"): - with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: - self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ - :6 - ] - return self._app_css_hash - - async def get_canned_queries(self, database_name, actor): - queries = ( - ((self.config or {}).get("databases") or {}).get(database_name) or {} - ).get("queries") or {} - for more_queries in pm.hook.canned_queries( - datasette=self, - database=database_name, - actor=actor, - ): - more_queries = await await_me_maybe(more_queries) - queries.update(more_queries or {}) - # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}} - for key in queries: - if not isinstance(queries[key], dict): - queries[key] = {"sql": queries[key]} - # Also make sure "name" is available: - queries[key]["name"] = key - return queries - - async def get_canned_query(self, database_name, query_name, actor): - queries = await self.get_canned_queries(database_name, actor) - query = queries.get(query_name) - if query: - return query + return self.static_hash("app.css") def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row @@ -942,14 +1493,14 @@ class Datasette: if request: actor = request.actor # Top-level link - if await self.permission_allowed(actor=actor, action="view-instance"): + if await self.allowed(action="view-instance", actor=actor): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link if database: - if await self.permission_allowed( - actor=actor, + if await self.allowed( action="view-database", - resource=database, + resource=DatabaseResource(database=database), + actor=actor, ): crumbs.append( { @@ -960,10 +1511,10 @@ class Datasette: # Table link if table: assert database, "table= requires database=" - if await self.permission_allowed( - actor=actor, + if await self.allowed( action="view-table", - resource=(database, table), + resource=TableResource(database=database, table=table), + actor=actor, ): crumbs.append( { @@ -974,8 +1525,8 @@ class Datasette: return crumbs async def actors_from_ids( - self, actor_ids: Iterable[Union[str, int]] - ) -> Dict[Union[id, str], Dict]: + self, actor_ids: Iterable[str | int] + ) -> Dict[int | str, Dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -990,258 +1541,355 @@ class Datasette: for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) - async def permission_allowed( - self, actor, action, resource=None, *, default=DEFAULT_NOT_SET - ): - """Check permissions using the permissions_allowed plugin hook""" - result = None - # Use default from registered permission, if available - if default is DEFAULT_NOT_SET and action in self.permissions: - default = self.permissions[action].default - opinions = [] - # Every plugin is consulted for their opinion - for check in pm.hook.permission_allowed( - datasette=self, - actor=actor, - action=action, - resource=resource, - ): - check = await await_me_maybe(check) - if check is not None: - opinions.append(check) - - result = None - # If any plugin said False it's false - the veto rule - if any(not r for r in opinions): - result = False - elif any(r for r in opinions): - # Otherwise, if any plugin said True it's true - result = True - - used_default = False - if result is None: - # No plugin expressed an opinion, so use the default - result = default - used_default = True - self._permission_checks.append( - { - "when": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "actor": actor, - "action": action, - "resource": resource, - "used_default": used_default, - "result": result, - } - ) - return result - - async def allowed_resources_sql( - self, actor: dict | None, action: str - ) -> tuple[str, dict]: - """Combine permission_resources_sql PluginSQL blocks into a UNION query. - - Returns a (sql, params) tuple suitable for execution against SQLite. + def resource_for_action(self, action: str, parent: str | None, child: str | None): """ - plugin_blocks: List[PluginSQL] = [] - for block in pm.hook.permission_resources_sql( - datasette=self, - actor=actor, - action=action, - ): - block = await await_me_maybe(block) - if block is None: - continue - if isinstance(block, (list, tuple)): - candidates = block - else: - candidates = [block] - for candidate in candidates: - if candidate is None: - continue - if not isinstance(candidate, PluginSQL): - continue - plugin_blocks.append(candidate) + Create a Resource instance for the given action with parent/child values. - actor_id = actor.get("id") if actor else None - sql, params = build_rules_union( - actor=str(actor_id) if actor_id is not None else "", - plugins=plugin_blocks, - ) - return sql, params + Looks up the action's resource_class and instantiates it with the + provided parent and child identifiers. - async def permission_allowed_2( - self, actor, action, resource=None, *, default=DEFAULT_NOT_SET - ): - """Permission check backed by permission_resources_sql rules.""" + Args: + action: The action name (e.g., "view-table", "view-query") + parent: The parent resource identifier (e.g., database name) + child: The child resource identifier (e.g., table/query name) - if default is DEFAULT_NOT_SET and action in self.permissions: - default = self.permissions[action].default + Returns: + A Resource instance of the appropriate subclass - if isinstance(actor, dict) or actor is None: - actor_dict = actor - else: - actor_dict = {"id": actor} - actor_id = actor_dict.get("id") if actor_dict else None - - candidate_parent = None - candidate_child = None - if isinstance(resource, str): - candidate_parent = resource - elif isinstance(resource, (tuple, list)) and len(resource) == 2: - candidate_parent, candidate_child = resource - elif resource is not None: - raise TypeError("resource must be None, str, or (parent, child) tuple") - - union_sql, union_params = await self.allowed_resources_sql(actor_dict, action) - - query = f""" - WITH rules AS ( - {union_sql} - ), - candidate AS ( - SELECT :cand_parent AS parent, :cand_child AS child - ), - matched AS ( - SELECT - r.allow, - r.reason, - r.source_plugin, - CASE - WHEN r.child IS NOT NULL THEN 2 - WHEN r.parent IS NOT NULL THEN 1 - ELSE 0 - END AS depth - FROM rules r - JOIN candidate c - ON (r.parent IS NULL OR r.parent = c.parent) - AND (r.child IS NULL OR r.child = c.child) - ), - ranked AS ( - SELECT *, - ROW_NUMBER() OVER ( - ORDER BY - depth DESC, - CASE WHEN allow = 0 THEN 0 ELSE 1 END, - source_plugin - ) AS rn - FROM matched - ), - winner AS ( - SELECT allow, reason, source_plugin, depth - FROM ranked - WHERE rn = 1 - ) - SELECT allow, reason, source_plugin, depth FROM winner + Raises: + ValueError: If the action is unknown """ + from datasette.permissions import Resource - params = { - **union_params, - "cand_parent": candidate_parent, - "cand_child": candidate_child, - } + action_obj = self.actions.get(action) + if not action_obj: + raise ValueError(f"Unknown action: {action}") - rows = await self.get_internal_database().execute(query, params) - row = rows.first() - - reason = None - source_plugin = None - depth = None - used_default = False - - if row is None: - result = default - used_default = True - else: - allow = row["allow"] - reason = row["reason"] - source_plugin = row["source_plugin"] - depth = row["depth"] - if allow is None: - result = default - used_default = True - else: - result = bool(allow) - - self._permission_checks.append( - { - "when": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "actor": actor, - "action": action, - "resource": resource, - "used_default": used_default, - "result": result, - "reason": reason, - "source_plugin": source_plugin, - "depth": depth, - } - ) - - return result - - async def ensure_permissions( - self, - actor: dict, - permissions: Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]], - ): - """ - permissions is a list of (action, resource) tuples or 'action' strings - - Raises datasette.Forbidden() if any of the checks fail - """ - assert actor is None or isinstance(actor, dict), "actor must be None or a dict" - for permission in permissions: - if isinstance(permission, str): - action = permission - resource = None - elif isinstance(permission, (tuple, list)) and len(permission) == 2: - action, resource = permission - else: - assert ( - False - ), "permission should be string or tuple of two items: {}".format( - repr(permission) - ) - ok = await self.permission_allowed( - actor, - action, - resource=resource, - default=None, - ) - if ok is not None: - if ok: - return - else: - raise Forbidden(action) + resource_class = action_obj.resource_class + instance = object.__new__(resource_class) + Resource.__init__(instance, parent=parent, child=child) + return instance async def check_visibility( self, actor: dict, - action: Optional[str] = None, - resource: Optional[Union[str, Tuple[str, str]]] = None, - permissions: Optional[ - Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]] - ] = None, + action: str, + resource: "Resource" | None = None, ): - """Returns (visible, private) - visible = can you see it, private = can others see it too""" - if permissions: - assert ( - not action and not resource - ), "Can't use action= or resource= with permissions=" - else: - permissions = [(action, resource)] - try: - await self.ensure_permissions(actor, permissions) - except Forbidden: + """ + Check if actor can see a resource and if it's private. + + Returns (visible, private) tuple: + - visible: bool - can the actor see it? + - private: bool - if visible, can anonymous users NOT see it? + """ + from datasette.permissions import Resource + + # Validate that resource is a Resource object or None + if resource is not None and not isinstance(resource, Resource): + raise TypeError("resource must be a Resource subclass instance or None.") + + # Check if actor can see it + if not await self.allowed(action=action, resource=resource, actor=actor): return False, False - # User can see it, but can the anonymous user see it? - try: - await self.ensure_permissions(None, permissions) - except Forbidden: - # It's visible but private + + # Check if anonymous user can see it (for "private" flag) + if not await self.allowed(action=action, resource=resource, actor=None): + # Actor can see it but anonymous cannot - it's private return True, True - # It's visible to everyone + + # Both actor and anonymous can see it - it's public return True, False + async def allowed_resources_sql( + self, + *, + action: str, + actor: dict | None = None, + parent: str | None = None, + include_is_private: bool = False, + ) -> ResourcesSQL: + """ + Build SQL query to get all resources the actor can access for the given action. + + Args: + action: The action name (e.g., "view-table") + actor: The actor dict (or None for unauthenticated) + parent: Optional parent filter (e.g., database name) to limit results + include_is_private: If True, include is_private column showing if anonymous cannot access + + Returns a namedtuple of (query: str, params: dict) that can be executed against the internal database. + The query returns rows with (parent, child, reason) columns, plus is_private if requested. + + Example: + query, params = await datasette.allowed_resources_sql( + action="view-table", + actor=actor, + parent="mydb", + include_is_private=True + ) + result = await datasette.get_internal_database().execute(query, params) + """ + from datasette.utils.actions_sql import build_allowed_resources_sql + + action_obj = self.actions.get(action) + if not action_obj: + raise ValueError(f"Unknown action: {action}") + + sql, params = await build_allowed_resources_sql( + self, actor, action, parent=parent, include_is_private=include_is_private + ) + return ResourcesSQL(sql, params) + + async def allowed_resources( + self, + action: str, + actor: dict | None = None, + *, + parent: str | None = None, + include_is_private: bool = False, + include_reasons: bool = False, + limit: int = 100, + next: str | None = None, + ) -> PaginatedResources: + """ + Return paginated resources the actor can access for the given action. + + Uses SQL with keyset pagination to efficiently filter resources. + Returns PaginatedResources with list of Resource instances and pagination metadata. + + Args: + action: The action name (e.g., "view-table") + actor: The actor dict (or None for unauthenticated) + parent: Optional parent filter (e.g., database name) to limit results + include_is_private: If True, adds a .private attribute to each Resource + include_reasons: If True, adds a .reasons attribute with List[str] of permission reasons + limit: Maximum number of results to return (1-1000, default 100) + next: Keyset token from previous page for pagination + + Returns: + PaginatedResources with: + - resources: List of Resource objects for this page + - next: Token for next page (None if no more results) + + Example: + # Get first page of tables + page = await datasette.allowed_resources("view-table", actor, limit=50) + for table in page.resources: + print(f"{table.parent}/{table.child}") + + # Get next page + if page.next: + next_page = await datasette.allowed_resources( + "view-table", actor, limit=50, next=page.next + ) + + # With reasons for debugging + page = await datasette.allowed_resources( + "view-table", actor, include_reasons=True + ) + for table in page.resources: + print(f"{table.child}: {table.reasons}") + + # Iterate through all results with async generator + page = await datasette.allowed_resources("view-table", actor) + async for table in page.all(): + print(table.child) + """ + + action_obj = self.actions.get(action) + if not action_obj: + raise ValueError(f"Unknown action: {action}") + + # Validate and cap limit + limit = min(max(1, limit), 1000) + + # Get base SQL query + query, params = await self.allowed_resources_sql( + action=action, + actor=actor, + parent=parent, + include_is_private=include_is_private, + ) + + # Add keyset pagination WHERE clause if next token provided + if next: + try: + components = urlsafe_components(next) + if len(components) >= 2: + last_parent, last_child = components[0], components[1] + # Keyset condition: (parent > last) OR (parent = last AND child > last) + keyset_where = """ + (parent > :keyset_parent OR + (parent = :keyset_parent AND child > :keyset_child)) + """ + # Wrap original query and add keyset filter + query = f"SELECT * FROM ({query}) WHERE {keyset_where}" + params["keyset_parent"] = last_parent + params["keyset_child"] = last_child + except (ValueError, KeyError): + # Invalid token - ignore and start from beginning + pass + + # Add LIMIT (fetch limit+1 to detect if there are more results) + # Note: query from allowed_resources_sql() already includes ORDER BY parent, child + query = f"{query} LIMIT :limit" + params["limit"] = limit + 1 + + # Execute query + result = await self.get_internal_database().execute(query, params) + rows = list(result.rows) + + # Check if truncated (got more than limit rows) + truncated = len(rows) > limit + if truncated: + rows = rows[:limit] # Remove the extra row + + # Build Resource objects with optional attributes + resources = [] + for row in rows: + # row[0]=parent, row[1]=child, row[2]=reason, row[3]=is_private (if requested) + resource = self.resource_for_action(action, parent=row[0], child=row[1]) + + # Add reasons if requested + if include_reasons: + reason_json = row[2] + try: + reasons_array = ( + json.loads(reason_json) if isinstance(reason_json, str) else [] + ) + resource.reasons = [r for r in reasons_array if r is not None] + except (json.JSONDecodeError, TypeError): + resource.reasons = [reason_json] if reason_json else [] + + # Add private flag if requested + if include_is_private: + resource.private = bool(row[3]) + + resources.append(resource) + + # Generate next token if there are more results + next_token = None + if truncated and resources: + last_resource = resources[-1] + # Use tilde-encoding like table pagination + next_token = "{},{}".format( + tilde_encode(str(last_resource.parent)), + tilde_encode(str(last_resource.child)), + ) + + return PaginatedResources( + resources=resources, + next=next_token, + _datasette=self, + _action=action, + _actor=actor, + _parent=parent, + _include_is_private=include_is_private, + _include_reasons=include_reasons, + _limit=limit, + ) + + async def allowed( + self, + *, + action: str, + resource: "Resource" = None, + actor: dict | None = None, + ) -> bool: + """ + Check if actor can perform action on specific resource. + + Uses SQL to check permission for a single resource without fetching all resources. + This is efficient - it does NOT call allowed_resources() and check membership. + + For global actions, resource should be None (or omitted). + + Example: + from datasette.resources import TableResource + can_view = await datasette.allowed( + action="view-table", + resource=TableResource(database="analytics", table="users"), + actor=actor + ) + + # For global actions, resource can be omitted: + can_debug = await datasette.allowed(action="permissions-debug", actor=actor) + """ + from datasette.utils.actions_sql import check_permission_for_resource + + # For global actions, resource remains None + + # Check if this action has also_requires - if so, check that action first + action_obj = self.actions.get(action) + if action_obj and action_obj.also_requires: + # Must have the required action first + if not await self.allowed( + action=action_obj.also_requires, + resource=resource, + actor=actor, + ): + return False + + # For global actions, resource is None + parent = resource.parent if resource else None + child = resource.child if resource else None + + result = await check_permission_for_resource( + datasette=self, + actor=actor, + action=action, + parent=parent, + child=child, + ) + + # Log the permission check for debugging + self._permission_checks.append( + PermissionCheck( + when=datetime.datetime.now(datetime.timezone.utc).isoformat(), + actor=actor, + action=action, + parent=parent, + child=child, + result=result, + ) + ) + + return result + + async def ensure_permission( + self, + *, + action: str, + resource: "Resource" = None, + actor: dict | None = None, + ): + """ + Check if actor can perform action on resource, raising Forbidden if not. + + This is a convenience wrapper around allowed() that raises Forbidden + instead of returning False. Use this when you want to enforce a permission + check and halt execution if it fails. + + Example: + from datasette.resources import TableResource + + # Will raise Forbidden if actor cannot view the table + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database="analytics", table="users"), + actor=request.actor + ) + + # For instance-level actions, resource can be omitted: + await datasette.ensure_permission( + action="permissions-debug", + actor=request.actor + ) + """ + if not await self.allowed(action=action, resource=resource, actor=actor): + raise Forbidden(action) + async def execute( self, db_name, @@ -1276,15 +1924,14 @@ class Datasette: except IndexError: return {} # Ensure user has permission to view the referenced table + from datasette.resources import TableResource + other_table = fk["other_table"] other_column = fk["other_column"] visible, _ = await self.check_visibility( actor, - permissions=[ - ("view-table", (database, other_table)), - ("view-database", database), - "view-instance", - ], + action="view-table", + resource=TableResource(database=database, table=other_table), ) if not visible: return {} @@ -1404,6 +2051,7 @@ class Datasette: break except importlib.metadata.PackageNotFoundError: pass + conn.close() return info def _plugins(self, request=None, all=False): @@ -1449,6 +2097,22 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} + def _actions(self): + return [ + { + "name": action.name, + "abbr": action.abbr, + "description": action.description, + "takes_parent": action.takes_parent, + "takes_child": action.takes_child, + "resource_class": ( + action.resource_class.__name__ if action.resource_class else None + ), + "also_requires": action.also_requires, + } + for action in sorted(self.actions.values(), key=lambda a: a.name) + ] + async def table_config(self, database: str, table: str) -> dict: """Return dictionary of configuration for specified table""" return ( @@ -1482,10 +2146,10 @@ class Datasette: async def render_template( self, - templates: Union[List[str], str, Template], - context: Optional[Union[Dict[str, Any], Context]] = None, - request: Optional[Request] = None, - view_name: Optional[str] = None, + templates: List[str] | str | Template, + context: Dict[str, Any] | Context | None = None, + request: Request | None = None, + view_name: str | None = None, ): if not self._startup_invoked: raise Exception("render_template() called before await ds.invoke_startup()") @@ -1571,7 +2235,11 @@ class Datasette: "extra_js_urls", template, context, request, view_name ), "base_url": self.setting("base_url"), - "csrftoken": request.scope["csrftoken"] if request else lambda: "", + "csrftoken": ( + request.scope["csrftoken"] + if request and "csrftoken" in request.scope + else lambda: "" + ), "datasette_version": __version__, }, **extra_template_vars, @@ -1584,7 +2252,7 @@ class Datasette: return await template.render_async(template_context) def set_actor_cookie( - self, response: Response, actor: dict, expire_after: Optional[int] = None + self, response: Response, actor: dict, expire_after: int | None = None ): data = {"a": actor} if expire_after: @@ -1714,6 +2382,16 @@ class Datasette: ), r"/-/actor(\.(?Pjson))?$", ) + add_route( + JsonDataView.as_view( + self, + "actions.json", + self._actions, + template="debug_actions.html", + permission="permissions-debug", + ), + r"/-/actions(\.(?Pjson))?$", + ) add_route( AuthTokenView.as_view(self), r"/-/auth-token$", @@ -1726,6 +2404,18 @@ class Datasette: ApiExplorerView.as_view(self), r"/-/api$", ) + add_route( + JumpView.as_view(self), + r"/-/jump(\.(?Pjson))?$", + ) + add_route( + GlobalQueryListView.as_view(self), + r"/-/queries(\.(?Pjson))?$", + ) + add_route( + InstanceSchemaView.as_view(self), + r"/-/schema(\.(?Pjson|md))?$", + ) add_route( LogoutView.as_view(self), r"/-/logout$", @@ -1767,10 +2457,50 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") + add_route( + QueryListView.as_view(self), + r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", + ) + add_route( + QueryCreateAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/analyze$", + ) + add_route( + QueryStoreView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/store$", + ) + add_route( + ExecuteWriteAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/execute-write/analyze$", + ) + add_route( + ExecuteWriteView.as_view(self), + r"/(?P[^\/\.]+)/-/execute-write$", + ) + add_route( + DatabaseSchemaView.as_view(self), + r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", + ) + add_route( + QueryParametersView.as_view(self), + r"/(?P[^\/\.]+)/-/query/parameters$", + ) add_route( wrap_view(QueryView, self), r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", ) + add_route( + QueryDefinitionView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", + ) + add_route( + QueryUpdateView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", + ) + add_route( + QueryDeleteView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/delete$", + ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", @@ -1787,10 +2517,18 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/upsert$", ) + add_route( + TableSetColumnTypeView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", + ) add_route( TableDropView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", ) + add_route( + TableSchemaView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/schema(\.(?Pjson|md))?$", + ) add_route( RowDeleteView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/delete$", @@ -1844,29 +2582,13 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) - async def custom_csrf_error(scope, send, message_id): - await asgi_send( - send, - content=await self.render_template( - "csrf_error.html", - {"message_id": message_id, "message_name": Errors(message_id).name}, - ), - status=403, - content_type="text/html; charset=utf-8", - ) + async def _close_on_shutdown(): + self.close() - asgi = asgi_csrf.asgi_csrf( - DatasetteRouter(self, routes), - signing_secret=self._secret, - cookie_name="ds_csrftoken", - skip_if_scope=lambda scope: any( - pm.hook.skip_csrf(datasette=self, scope=scope) - ), - send_csrf_failed=custom_csrf_error, - ) + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) @@ -1913,10 +2635,13 @@ class DatasetteRouter: # Handle authentication default_actor = scope.get("actor") or None actor = None - for actor in pm.hook.actor_from_request(datasette=self.ds, request=request): - actor = await await_me_maybe(actor) - if actor: - break + results = pm.hook.actor_from_request(datasette=self.ds, request=request) + for result in results: + result = await await_me_maybe(result) + if result and actor is None: + actor = result + # Don't break — we must await all coroutines to avoid + # "coroutine was never awaited" warnings scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) @@ -2181,9 +2906,18 @@ class NotFoundExplicit(NotFound): class DatasetteClient: + """Internal HTTP client for making requests to a Datasette instance. + + Used for testing and for internal operations that need to make HTTP requests + to the Datasette app without going through an actual HTTP server. + """ + def __init__(self, ds): self.ds = ds - self.app = ds.app() + + @property + def app(self): + return self.ds.app() def actor_cookie(self, actor): # Utility method, mainly for tests @@ -2196,40 +2930,102 @@ class DatasetteClient: path = f"http://localhost{path}" return path - async def _request(self, method, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await getattr(client, method)(self._fix(path), **kwargs) + def _apply_actor(self, kwargs): + """If ``actor=`` was supplied, convert it into a signed ds_actor cookie.""" + actor = kwargs.pop("actor", None) + if actor is None: + return + cookies = dict(kwargs.get("cookies") or {}) + if "ds_actor" in cookies: + raise TypeError("Cannot pass both actor= and a ds_actor cookie") + cookies["ds_actor"] = self.actor_cookie(actor) + kwargs["cookies"] = cookies - async def get(self, path, **kwargs): - return await self._request("get", path, **kwargs) + async def _request(self, method, path, skip_permission_checks=False, **kwargs): + from datasette.permissions import SkipPermissions - async def options(self, path, **kwargs): - return await self._request("options", path, **kwargs) + self._apply_actor(kwargs) + with _DatasetteClientContext(): + if skip_permission_checks: + with SkipPermissions(): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), + ) as client: + return await getattr(client, method)(self._fix(path), **kwargs) + else: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), + ) as client: + return await getattr(client, method)(self._fix(path), **kwargs) - async def head(self, path, **kwargs): - return await self._request("head", path, **kwargs) + async def get(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "get", path, skip_permission_checks=skip_permission_checks, **kwargs + ) - async def post(self, path, **kwargs): - return await self._request("post", path, **kwargs) + async def options(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "options", path, skip_permission_checks=skip_permission_checks, **kwargs + ) - async def put(self, path, **kwargs): - return await self._request("put", path, **kwargs) + async def head(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "head", path, skip_permission_checks=skip_permission_checks, **kwargs + ) - async def patch(self, path, **kwargs): - return await self._request("patch", path, **kwargs) + async def post(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "post", path, skip_permission_checks=skip_permission_checks, **kwargs + ) - async def delete(self, path, **kwargs): - return await self._request("delete", path, **kwargs) + async def put(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "put", path, skip_permission_checks=skip_permission_checks, **kwargs + ) + + async def patch(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "patch", path, skip_permission_checks=skip_permission_checks, **kwargs + ) + + async def delete(self, path, skip_permission_checks=False, **kwargs): + return await self._request( + "delete", path, skip_permission_checks=skip_permission_checks, **kwargs + ) + + async def request(self, method, path, skip_permission_checks=False, **kwargs): + """Make an HTTP request with the specified method. + + Args: + method: HTTP method (e.g., "GET", "POST", "PUT") + path: The path to request + skip_permission_checks: If True, bypass all permission checks for this request + **kwargs: Additional arguments to pass to httpx + + Returns: + httpx.Response: The response from the request + """ + from datasette.permissions import SkipPermissions - async def request(self, method, path, **kwargs): avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.request( - method, self._fix(path, avoid_path_rewrites), **kwargs - ) + self._apply_actor(kwargs) + with _DatasetteClientContext(): + if skip_permission_checks: + with SkipPermissions(): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), + ) as client: + return await client.request( + method, self._fix(path, avoid_path_rewrites), **kwargs + ) + else: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), + ) as client: + return await client.request( + method, self._fix(path, avoid_path_rewrites), **kwargs + ) diff --git a/datasette/cli.py b/datasette/cli.py index bacabc4c..90a33e80 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .app import ( SQLITE_LIMIT_ATTACHED, pm, ) +from .inspect import inspect_tables from .utils import ( LoadExtension, StartupError, @@ -109,15 +110,11 @@ def sqlite_extensions(fn): return fn(*args, **kwargs) except AttributeError as e: if "enable_load_extension" in str(e): - raise click.ClickException( - textwrap.dedent( - """ + raise click.ClickException(textwrap.dedent(""" Your Python installation does not have the ability to load SQLite extensions. More information: https://datasette.io/help/extensions - """ - ).strip() - ) + """).strip()) raise return wrapped @@ -146,7 +143,6 @@ def inspect(files, inspect_file, sqlite_extensions): This can then be passed to "datasette --inspect-file" to speed up count operations against immutable database files. """ - app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) inspect_data = run_sync(lambda: inspect_(files, sqlite_extensions)) if inspect_file == "-": sys.stdout.write(json.dumps(inspect_data, indent=2)) @@ -159,14 +155,14 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - counts = await database.table_counts(limit=3600 * 1000) + tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) data[name] = { "hash": database.hash, "size": database.size, "file": database.path, "tables": { - table_name: {"count": table_count} - for table_name, table_count in counts.items() + table_name: {"count": table["count"]} + for table_name, table in tables.items() }, } return data @@ -439,10 +435,20 @@ def uninstall(packages, yes): help="Output URL that sets a cookie authenticating the root user", is_flag=True, ) +@click.option( + "--default-deny", + help="Deny all permissions by default", + is_flag=True, +) @click.option( "--get", help="Run an HTTP GET request against this path, print results and exit", ) +@click.option( + "--headers", + is_flag=True, + help="Include HTTP headers in --get output", +) @click.option( "--token", help="API token to send with --get requests", @@ -510,7 +516,9 @@ def serve( settings, secret, root, + default_deny, get, + headers, token, actor, version_note, @@ -540,7 +548,7 @@ def serve( if reload: import hupper - reloader = hupper.start_reloader("datasette.cli.serve") + reloader = hupper.start_reloader("datasette.cli.cli") if immutable: reloader.watch_files(immutable) if config: @@ -589,18 +597,28 @@ def serve( crossdb=crossdb, nolock=nolock, internal=internal, + default_deny=default_deny, ) - # if files is a single directory, use that as config_dir= - if 1 == len(files) and os.path.isdir(files[0]): - kwargs["config_dir"] = pathlib.Path(files[0]) - files = [] + # Separate directories from files + directories = [f for f in files if os.path.isdir(f)] + file_paths = [f for f in files if not os.path.isdir(f)] + + # Handle config_dir - only one directory allowed + if len(directories) > 1: + raise click.ClickException( + "Cannot pass multiple directories. Pass a single directory as config_dir." + ) + elif len(directories) == 1: + kwargs["config_dir"] = pathlib.Path(directories[0]) # Verify list of files, create if needed (and --create) - for file in files: + for file in file_paths: if not pathlib.Path(file).exists(): if create: - sqlite3.connect(file).execute("vacuum") + conn = sqlite3.connect(file) + conn.execute("vacuum") + conn.close() else: raise click.ClickException( "Invalid value for '[FILES]...': Path '{}' does not exist.".format( @@ -608,8 +626,32 @@ def serve( ) ) - # De-duplicate files so 'datasette db.db db.db' only attaches one /db - files = list(dict.fromkeys(files)) + # Check for duplicate files by resolving all paths to their absolute forms + # Collect all database files that will be loaded (explicit files + config_dir files) + all_db_files = [] + + # Add explicit files + for file in file_paths: + all_db_files.append((file, pathlib.Path(file).resolve())) + + # Add config_dir databases if config_dir is set + if "config_dir" in kwargs: + config_dir = kwargs["config_dir"] + for ext in ("db", "sqlite", "sqlite3"): + for db_file in config_dir.glob(f"*.{ext}"): + all_db_files.append((str(db_file), db_file.resolve())) + + # Check for duplicates + seen = {} + for original_path, resolved_path in all_db_files: + if resolved_path in seen: + raise click.ClickException( + f"Duplicate database file: '{original_path}' and '{seen[resolved_path]}' " + f"both refer to {resolved_path}" + ) + seen[resolved_path] = original_path + + files = file_paths try: ds = Datasette(files, **kwargs) @@ -622,25 +664,43 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run the "startup" plugin hooks - run_sync(ds.invoke_startup) - - # Run async soundness checks - but only if we're not under pytest + # Run async soundness checks before startup hooks, since invoke_startup + # now populates internal tables which requires querying each database run_sync(lambda: check_databases(ds)) + # Run the "startup" plugin hooks + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + + if headers and not get: + raise click.ClickException("--headers can only be used with --get") + if token and not get: raise click.ClickException("--token can only be used with --get") if get: client = TestClient(ds) - headers = {} + request_headers = {} if token: - headers["Authorization"] = "Bearer {}".format(token) + request_headers["Authorization"] = "Bearer {}".format(token) cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) - response = client.get(get, headers=headers, cookies=cookies) - click.echo(response.text) + response = client.get(get, headers=request_headers, cookies=cookies) + + if headers: + # Output HTTP status code, headers, two newlines, then the response body + click.echo(f"HTTP/1.1 {response.status}") + for key, value in response.headers.items(): + click.echo(f"{key}: {value}") + if response.text: + click.echo() + click.echo(response.text) + else: + click.echo(response.text) + exit_code = 0 if response.status == 200 else 1 sys.exit(exit_code) return @@ -648,6 +708,7 @@ def serve( # Start the server url = None if root: + ds.root_enabled = True url = "http://{}:{}{}?token={}".format( host, port, ds.urls.path("-/auth-token"), ds._root_token ) @@ -757,7 +818,10 @@ def create_token( ds = Datasette(secret=secret, plugins_dir=plugins_dir) # Run ds.invoke_startup() in an event loop - run_sync(ds.invoke_startup) + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) # Warn about any unknown actions actions = [] @@ -765,28 +829,30 @@ def create_token( actions.extend([p[1] for p in databases]) actions.extend([p[2] for p in resources]) for action in actions: - if not ds.permissions.get(action): + if not ds.actions.get(action): click.secho( f" Unknown permission: {action} ", fg="red", err=True, ) - restrict_database = {} - for database, action in databases: - restrict_database.setdefault(database, []).append(action) - restrict_resource = {} - for database, resource, action in resources: - restrict_resource.setdefault(database, {}).setdefault(resource, []).append( - action - ) + from datasette.tokens import TokenRestrictions - token = ds.create_token( - id, - expires_after=expires_after, - restrict_all=alls, - restrict_database=restrict_database, - restrict_resource=restrict_resource, + restrictions = TokenRestrictions() + for action in alls: + restrictions.allow_all(action) + for database, action in databases: + restrictions.allow_database(database, action) + for database, resource, action in resources: + restrictions.allow_resource(database, resource, action) + + token = run_sync( + lambda: ds.create_token( + id, + expires_after=expires_after, + restrictions=restrictions, + handler="signed", + ) ) click.echo(token) if debug: diff --git a/datasette/column_types.py b/datasette/column_types.py new file mode 100644 index 00000000..7320e1d6 --- /dev/null +++ b/datasette/column_types.py @@ -0,0 +1,83 @@ +from enum import Enum + + +class SQLiteType(Enum): + TEXT = "TEXT" + INTEGER = "INTEGER" + REAL = "REAL" + BLOB = "BLOB" + NULL = "NULL" + + @classmethod + def from_declared_type(cls, declared_type: str | None) -> "SQLiteType | None": + if declared_type is None: + return cls.NULL + + normalized = declared_type.strip().upper() + if not normalized: + return cls.NULL + + if normalized == cls.NULL.value: + return cls.NULL + if "INT" in normalized: + return cls.INTEGER + if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")): + return cls.TEXT + if "BLOB" in normalized: + return cls.BLOB + if any( + token in normalized + for token in ("REAL", "FLOA", "DOUB") # codespell:ignore doub + ): + return cls.REAL + + return None + + +class ColumnType: + """ + Base class for column types. + + Subclasses must define ``name`` and ``description`` as class attributes: + + - ``name``: Unique identifier string. Lowercase, no spaces. + Examples: "markdown", "file", "email", "url", "point", "image". + - ``description``: Human-readable label for admin UI dropdowns. + Examples: "Markdown text", "File reference", "Email address". + - ``sqlite_types``: Optional tuple of SQLiteType values restricting + which SQLite column types this ColumnType can be assigned to. + + Instantiate with an optional ``config`` dict to bind per-column + configuration:: + + ct = MyColumnType(config={"key": "value"}) + ct.config # {"key": "value"} + """ + + name: str + description: str + sqlite_types: tuple[SQLiteType, ...] | None = None + + def __init__(self, config=None): + self.config = config + + async def render_cell(self, value, column, table, database, datasette, request): + """ + Return an HTML string to render this cell value, or None to + fall through to the default render_cell plugin hook chain. + """ + return None + + async def validate(self, value, datasette): + """ + Validate a value before it is written. Return None if valid, + or a string error message if invalid. + """ + return None + + async def transform_value(self, value, datasette): + """ + Transform a value before it appears in JSON API output. + Return the transformed value. Default: return unchanged. + """ + return value diff --git a/datasette/csrf.py b/datasette/csrf.py new file mode 100644 index 00000000..df239aee --- /dev/null +++ b/datasette/csrf.py @@ -0,0 +1,178 @@ +""" +Header-based CSRF (Cross-Origin) protection. + +Datasette uses the Sec-Fetch-Site + Origin header approach described in +Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented +in Go 1.25's http.CrossOriginProtection. This replaces the previous +token-based asgi-csrf mechanism. +""" + +from __future__ import annotations + +import secrets +import urllib.parse + +from .utils.asgi import asgi_send + +SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) + +DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443} + + +def _normalize_headers(raw_headers): + """Lowercase header names; for duplicates, last value wins.""" + result = {} + for name, value in raw_headers: + if isinstance(name, str): + name = name.encode("latin-1") + if isinstance(value, str): + value = value.encode("latin-1") + result[name.lower()] = value + return result + + +def _origin_tuple(value): + """ + Parse an origin-like string into ``(scheme, host, port)`` with default + ports filled in. Raises ``ValueError`` for malformed input. + """ + parsed = urllib.parse.urlsplit(value) + scheme = (parsed.scheme or "").lower() + host = (parsed.hostname or "").lower() + if not scheme or not host: + raise ValueError("missing scheme or host in {!r}".format(value)) + port = parsed.port # may raise ValueError on bad ports + if port is None: + port = DEFAULT_PORTS.get(scheme) + if port is None: + raise ValueError("unknown default port for scheme {!r}".format(scheme)) + return scheme, host, port + + +def _install_legacy_csrftoken(scope): + """ + Populate ``scope["csrftoken"]`` with a callable returning a per-request + random token. Provided for plugin compatibility only - core no longer + uses this value for CSRF enforcement. + """ + + def csrftoken(): + if "_datasette_legacy_csrftoken" not in scope: + scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32) + return scope["_datasette_legacy_csrftoken"] + + scope["csrftoken"] = csrftoken + + +class CrossOriginProtectionMiddleware: + """ + Modern CSRF protection using the Sec-Fetch-Site and Origin headers. + + Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's + http.CrossOriginProtection. See https://words.filippo.io/csrf/ + + Unsafe-method requests are allowed through only if they look same-origin. + Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin + and are passed through unchanged - CSRF is a browser-only attack. + """ + + SAFE_METHODS = SAFE_METHODS + + def __init__(self, app, datasette): + self.app = app + self.datasette = datasette + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + _install_legacy_csrftoken(scope) + + if scope.get("method", "GET") in self.SAFE_METHODS: + await self.app(scope, receive, send) + return + + headers = _normalize_headers(scope.get("headers") or []) + + authorization = headers.get(b"authorization", b"").decode("latin-1") + cookie_header = headers.get(b"cookie") + # Bearer-token requests are not ambient browser credentials, so they + # are not CSRF-vulnerable. Narrowly exempt them from the header check + # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; + # schemes like Basic or Digest can be browser-managed and ambient. + # If the request also carries a Cookie header, ambient cookie auth + # could be in play, so do NOT treat it as exempt. + if authorization and not cookie_header: + parts = authorization.split(None, 1) + if parts and parts[0].lower() == "bearer": + await self.app(scope, receive, send) + return + + origin_bytes = headers.get(b"origin") + sec_fetch_site_bytes = headers.get(b"sec-fetch-site") + host_bytes = headers.get(b"host", b"") + origin = origin_bytes.decode("latin-1") if origin_bytes else None + sec_fetch_site = ( + sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None + ) + host = host_bytes.decode("latin-1") + + # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS) + if sec_fetch_site is not None: + if sec_fetch_site in ("same-origin", "none"): + await self.app(scope, receive, send) + return + await self._forbid( + send, + "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( + sec_fetch_site + ), + ) + return + + # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.) + if origin is None: + await self.app(scope, receive, send) + return + + # Fallback for older browsers: Origin must match the request's own + # scheme + host + port. Compare full origin tuples, not host alone. + request_scheme = self._request_scheme(scope) + try: + origin_tuple = _origin_tuple(origin) + expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) + except ValueError: + await self._forbid( + send, + "Malformed Origin {!r} or Host {!r}".format(origin, host), + ) + return + + if origin_tuple == expected_tuple: + await self.app(scope, receive, send) + return + + await self._forbid( + send, + "Origin {!r} does not match Host {!r}".format(origin, host), + ) + + def _request_scheme(self, scope): + if self.datasette is not None: + try: + if self.datasette.setting("force_https_urls"): + return "https" + except Exception: + pass + return scope.get("scheme") or "http" + + async def _forbid(self, send, reason): + await asgi_send( + send, + content=await self.datasette.render_template( + "csrf_error.html", {"reason": reason} + ), + status=403, + content_type="text/html; charset=utf-8", + ) diff --git a/datasette/database.py b/datasette/database.py index b74f02bb..10417670 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,15 +1,19 @@ import asyncio +import atexit from collections import namedtuple +import inspect +import os from pathlib import Path -import janus import queue import sqlite_utils import sys +import tempfile import threading import uuid from .tracer import trace from .utils import ( + call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, @@ -21,7 +25,8 @@ from .utils import ( table_columns, table_column_details, ) -from .utils.sqlite import sqlite_version +from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables +from .utils.sqlite import sqlite_hidden_table_names from .inspect import inspect_hash connections = threading.local() @@ -29,6 +34,13 @@ connections = threading.local() AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) +class DatasetteClosedError(RuntimeError): + """Raised when using a Datasette or Database instance after close().""" + + +_SHUTDOWN = object() + + class Database: # For table counts stop at this many rows: count_limit = 10000 @@ -42,6 +54,7 @@ class Database: is_memory=False, memory_name=None, mode=None, + is_temp_disk=False, ): self.name = None self._thread_local_id = f"x{self._thread_local_id_counter}" @@ -52,19 +65,44 @@ class Database: self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name + self.is_temp_disk = is_temp_disk if memory_name is not None: self.is_memory = True + if is_temp_disk: + fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_") + os.close(fd) + self.path = temp_path + self.is_mutable = True + self.mode = "rwc" + self._wal_enabled = False + atexit.register(self._cleanup_temp_file) + else: + self._wal_enabled = False self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None self._write_queue = None + self._closed = False + self._pending_execute_futures = set() + self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] - self.mode = mode + if not is_temp_disk: + self.mode = mode + + def _check_not_closed(self): + if self._closed: + raise DatasetteClosedError( + "Database {!r} has been closed".format(self.name) + ) + + def _remove_pending_execute_future(self, future): + with self._pending_execute_futures_lock: + self._pending_execute_futures.discard(future) @property def cached_table_counts(self): @@ -85,6 +123,8 @@ class Database: return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): + if self.is_temp_disk: + return "_temp_disk" if self.path: return Path(self.path).stem elif self.memory_name: @@ -123,30 +163,104 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) self._all_file_connections.append(conn) + if self.is_temp_disk and not self._wal_enabled: + conn.execute("PRAGMA journal_mode=WAL") + self._wal_enabled = True return conn def close(self): - # Close all connections - useful to avoid running out of file handles in tests - for connection in self._all_file_connections: - connection.close() + """Release all resources held by this database. + + Idempotent. After close() further calls to execute()/execute_fn()/ + execute_write()/execute_write_fn() raise DatasetteClosedError. + """ + if self._closed: + return + with self._pending_execute_futures_lock: + if self._closed: + return + self._closed = True + pending_execute_futures = tuple(self._pending_execute_futures) + # Shut down the write thread, if any, via a sentinel. The thread + # drains any writes already queued before the sentinel and then + # closes its own write connection and returns. + write_thread = self._write_thread + if write_thread is not None and self._write_queue is not None: + self._write_queue.put(_SHUTDOWN) + write_thread.join(timeout=10) + if write_thread.is_alive(): + sys.stderr.write( + "Datasette: write thread for {!r} did not exit within 10s\n".format( + self.name + ) + ) + sys.stderr.flush() + for future in pending_execute_futures: + try: + future.result() + except Exception: + pass + # Close anything still tracked in _all_file_connections + for connection in self._all_file_connections: + try: + connection.close() + except Exception: + pass + self._all_file_connections = [] + # Drop per-thread cached read connections we can reach + try: + delattr(connections, self._thread_local_id) + except AttributeError: + pass + # Close non-threaded-mode cached connections if still open + if self._read_connection is not None: + try: + self._read_connection.close() + except Exception: + pass + self._read_connection = None + if self._write_connection is not None: + try: + self._write_connection.close() + except Exception: + pass + self._write_connection = None + if self.is_temp_disk: + self._cleanup_temp_file() + + def _cleanup_temp_file(self): + if self.is_temp_disk and self.path: + for suffix in ("", "-wal", "-shm"): + try: + os.unlink(self.path + suffix) + except OSError: + pass + + async def execute_write(self, sql, params=None, block=True, request=None): + self._check_not_closed() - async def execute_write(self, sql, params=None, block=True): def _inner(conn): return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn(_inner, block=block) + results = await self.execute_write_fn(_inner, block=block, request=request) return results - async def execute_write_script(self, sql, block=True): + async def execute_write_script(self, sql, block=True, request=None): + self._check_not_closed() + def _inner(conn): return conn.executescript(sql) with trace("sql", database=self.name, sql=sql.strip(), executescript=True): - results = await self.execute_write_fn(_inner, block=block) + results = await self.execute_write_fn( + _inner, block=block, transaction=False, request=request + ) return results - async def execute_write_many(self, sql, params_seq, block=True): + async def execute_write_many(self, sql, params_seq, block=True, request=None): + self._check_not_closed() + def _inner(conn): count = 0 @@ -161,11 +275,14 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - results, count = await self.execute_write_fn(_inner, block=block) + results, count = await self.execute_write_fn( + _inner, block=block, request=request + ) kwargs["count"] = count return results async def execute_isolated_fn(self, fn): + self._check_not_closed() # Open a new connection just for the duration of this function # blocking the write queue to avoid any writes occurring during it if self.ds.executor is None: @@ -185,7 +302,21 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) - async def execute_write_fn(self, fn, block=True, transaction=True): + async def analyze_sql(self, sql, params=None) -> SQLAnalysis: + self._check_not_closed() + + return await self.execute_isolated_fn( + lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) + ) + + async def execute_write_fn(self, fn, block=True, transaction=True, request=None): + self._check_not_closed() + pending_events = [] + + def track_event(event): + pending_events.append(event) + + fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event) if self.ds.executor is None: # non-threaded mode if self._write_connection is None: @@ -193,13 +324,67 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: - return fn(self._write_connection) + result = fn(self._write_connection) else: - return fn(self._write_connection) + result = fn(self._write_connection) else: - return await self._send_to_write_thread( + result = await self._send_to_write_thread( fn, block=block, transaction=transaction ) + if block: + for event in pending_events: + await self.ds.track_event(event) + else: + # For non-blocking writes, spawn a background task to + # dispatch events after the write thread completes + task_id, reply_future = result + + async def _dispatch_events_after_write(): + try: + await reply_future + except Exception: + # if the write failed, don't emit success events + return + for event in pending_events: + await self.ds.track_event(event) + + asyncio.ensure_future(_dispatch_events_after_write()) + result = task_id + return result + + def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): + from .plugins import pm + + # Wrap fn so it receives track_event if its signature supports it. + # Historically fn was called positionally, so any single-parameter + # name (conn, connection, db, ...) worked. Preserve that by only + # switching to keyword dependency injection when the callback + # explicitly opts in by declaring a `track_event` parameter. + original_fn = fn + + if "track_event" in inspect.signature(original_fn).parameters: + + def fn_with_track_event(conn): + return call_with_supported_arguments( + original_fn, conn=conn, track_event=track_event + ) + + fn = fn_with_track_event + + wrappers = pm.hook.write_wrapper( + datasette=self.ds, + database=self.name, + request=request, + transaction=transaction, + ) + wrappers = [w for w in wrappers if w is not None] + if not wrappers: + return fn + # Build the wrapped fn by nesting context manager generators. + # The first wrapper returned by pluggy is outermost. + for wrapper_factory in reversed(wrappers): + fn = _apply_write_wrapper(fn, wrapper_factory, track_event) + return fn async def _send_to_write_thread( self, fn, block=True, isolated_connection=False, transaction=True @@ -215,18 +400,15 @@ class Database: ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - reply_queue = janus.Queue() + loop = asyncio.get_running_loop() + reply_future = loop.create_future() self._write_queue.put( - WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) ) if block: - result = await reply_queue.async_q.get() - if isinstance(result, Exception): - raise result - else: - return result + return await reply_future else: - return task_id + return task_id, reply_future def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -240,38 +422,47 @@ class Database: conn_exception = e while True: task = self._write_queue.get() + if task is _SHUTDOWN: + if conn is not None: + try: + conn.close() + except Exception: + pass + return + exception = None + result = None if conn_exception is not None: - result = conn_exception + exception = conn_exception + elif task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass else: - if task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - else: - try: - if task.transaction: - with conn: - result = task.fn(conn) - else: + try: + if task.transaction: + with conn: result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - task.reply_queue.sync_q.put(result) + else: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) async def execute_fn(self, fn): + self._check_not_closed() if self.ds.executor is None: # non-threaded mode if self._read_connection is None: @@ -288,9 +479,12 @@ class Database: setattr(connections, self._thread_local_id, conn) return fn(conn) - return await asyncio.get_event_loop().run_in_executor( - self.ds.executor, in_thread - ) + with self._pending_execute_futures_lock: + self._check_not_closed() + future = self.ds.executor.submit(in_thread) + self._pending_execute_futures.add(future) + future.add_done_callback(self._remove_pending_execute_future) + return await asyncio.wrap_future(future) async def execute( self, @@ -302,6 +496,7 @@ class Database: log_sql_errors=True, ): """Executes sql against db_name in a thread""" + self._check_not_closed() page_size = page_size or self.ds.page_size def sql_operation_in_thread(conn): @@ -349,7 +544,7 @@ class Database: def hash(self): if self.cached_hash is not None: return self.cached_hash - elif self.is_mutable or self.is_memory: + elif self.is_mutable or self.is_memory or self.is_temp_disk: return None elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): self.cached_hash = self.ds.inspect_data[self.name]["hash"] @@ -408,7 +603,12 @@ class Database: # But SQLite prior to 3.16.0 doesn't support pragma functions results = await self.execute("PRAGMA database_list;") # {'seq': 0, 'name': 'main', 'file': ''} - return [AttachedDatabase(*row) for row in results.rows if row["seq"] > 0] + return [ + AttachedDatabase(*row) + for row in results.rows + # Filter out the SQLite internal "temp" database, refs #2557 + if row["seq"] > 0 and row["name"] != "temp" + ] async def table_exists(self, table): results = await self.execute( @@ -424,7 +624,7 @@ class Database: async def table_names(self): results = await self.execute( - "select name from sqlite_master where type='table'" + "select name from sqlite_master where type='table' order by name" ) return [r[0] for r in results.rows] @@ -502,98 +702,7 @@ class Database: t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] - if sqlite_version()[1] >= 37: - hidden_tables += [ - x[0] - for x in await self.execute( - """ - with shadow_tables as ( - select name - from pragma_table_list - where [type] = 'shadow' - order by name - ), - core_tables as ( - select name - from sqlite_master - WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - combined as ( - select name from shadow_tables - union all - select name from core_tables - ) - select name from combined order by 1 - """ - ) - ] - else: - hidden_tables += [ - x[0] - for x in await self.execute( - """ - WITH base AS ( - SELECT name - FROM sqlite_master - WHERE name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - fts_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config')) - ), - fts5_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%' - ), - fts5_shadow_tables AS ( - SELECT - printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name - FROM fts5_names - JOIN fts_suffixes - ), - fts3_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize')) - ), - fts3_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%' - OR sql LIKE '%VIRTUAL TABLE%USING FTS4%' - ), - fts3_shadow_tables AS ( - SELECT - printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name - FROM fts3_names - JOIN fts3_suffixes - ), - final AS ( - SELECT name FROM base - UNION ALL - SELECT name FROM fts5_shadow_tables - UNION ALL - SELECT name FROM fts3_shadow_tables - ) - SELECT name FROM final ORDER BY 1 - """ - ) - ] - # Also hide any FTS tables that have a content= argument - hidden_tables += [ - x[0] - for x in await self.execute( - """ - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%' - AND sql LIKE '%USING FTS%' - AND sql LIKE '%content=%' - """ - ) - ] + hidden_tables += await self.execute_fn(sqlite_hidden_table_names) has_spatialite = await self.execute_fn(detect_spatialite) if has_spatialite: @@ -612,16 +721,11 @@ class Database: "KNN", "KNN2", ] + [ - r[0] - for r in ( - await self.execute( - """ + r[0] for r in (await self.execute(""" select name from sqlite_master where name like "idx_%" and type = "table" - """ - ) - ).rows + """)).rows ] return hidden_tables @@ -663,6 +767,8 @@ class Database: tags.append("mutable") if self.is_memory: tags.append("memory") + if self.is_temp_disk: + tags.append("temp_disk") if self.hash: tags.append(f"hash={self.hash}") if self.size is not None: @@ -673,17 +779,90 @@ class Database: return f"" -class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") +def _apply_write_wrapper(fn, wrapper_factory, track_event): + """Apply a single write_wrapper context manager around fn. - def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): + ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally + ``track_event``, and returns a generator that yields exactly once. + Code before the yield runs before ``fn(conn)``, code after the yield + runs after. The result of ``fn(conn)`` is sent into the generator + via ``.send()``, and any exception raised by ``fn(conn)`` is thrown + via ``.throw()``. + """ + + def wrapped(conn): + gen = call_with_supported_arguments( + wrapper_factory, conn=conn, track_event=track_event + ) + # Advance to the yield point (run "before" code) + try: + next(gen) + except StopIteration: + # Generator didn't yield — just run fn unchanged + return fn(conn) + + # Execute the actual write + try: + result = fn(conn) + except Exception: + # Throw exception into generator so it can handle it + try: + gen.throw(*sys.exc_info()) + except StopIteration: + pass + # Re-raise the original exception + raise + else: + # Send the result back through the yield + try: + gen.send(result) + except StopIteration: + pass + return result + + return wrapped + + +class WriteTask: + __slots__ = ( + "fn", + "task_id", + "loop", + "reply_future", + "isolated_connection", + "transaction", + ) + + def __init__( + self, fn, task_id, loop, reply_future, isolated_connection, transaction + ): self.fn = fn self.task_id = task_id - self.reply_queue = reply_queue + self.loop = loop + self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction +def _deliver_write_result(task, result, exception): + # Called from the write thread. Delivers the result back to the + # awaiting coroutine on its event loop via call_soon_threadsafe. + def _set(): + if task.reply_future.done(): + # Awaiter was cancelled; nothing to do. + return + if exception is not None: + task.reply_future.set_exception(exception) + else: + task.reply_future.set_result(result) + + try: + task.loop.call_soon_threadsafe(_set) + except RuntimeError: + # Event loop has been closed; the awaiter is gone. + pass + + class QueryInterrupted(Exception): def __init__(self, e, sql, params): self.e = e diff --git a/datasette/default_actions.py b/datasette/default_actions.py new file mode 100644 index 00000000..2f78570b --- /dev/null +++ b/datasette/default_actions.py @@ -0,0 +1,133 @@ +from datasette import hookimpl +from datasette.permissions import Action +from datasette.resources import ( + DatabaseResource, + TableResource, + QueryResource, +) + + +@hookimpl +def register_actions(): + """Register the core Datasette actions.""" + return ( + # Global actions (no resource_class) + Action( + name="view-instance", + abbr="vi", + description="View Datasette instance", + ), + Action( + name="permissions-debug", + abbr="pd", + description="Access permission debug tool", + ), + Action( + name="debug-menu", + abbr="dm", + description="View debug menu items", + ), + # Database-level actions (parent-level) + Action( + name="view-database", + abbr="vd", + description="View database", + resource_class=DatabaseResource, + ), + Action( + name="view-database-download", + abbr="vdd", + description="Download database file", + resource_class=DatabaseResource, + also_requires="view-database", + ), + Action( + name="execute-sql", + abbr="es", + description="Execute read-only SQL queries", + resource_class=DatabaseResource, + also_requires="view-database", + ), + Action( + name="execute-write-sql", + abbr="ews", + description="Execute writable SQL queries", + resource_class=DatabaseResource, + also_requires="view-database", + ), + Action( + name="create-table", + abbr="ct", + description="Create tables", + resource_class=DatabaseResource, + ), + Action( + name="store-query", + abbr="sq", + description="Create stored queries", + resource_class=DatabaseResource, + also_requires="execute-sql", + ), + # Table-level actions (child-level) + Action( + name="view-table", + abbr="vt", + description="View table", + resource_class=TableResource, + ), + Action( + name="insert-row", + abbr="ir", + description="Insert rows", + resource_class=TableResource, + ), + Action( + name="delete-row", + abbr="dr", + description="Delete rows", + resource_class=TableResource, + ), + Action( + name="update-row", + abbr="ur", + description="Update rows", + resource_class=TableResource, + ), + Action( + name="alter-table", + abbr="at", + description="Alter tables", + resource_class=TableResource, + ), + Action( + name="set-column-type", + abbr="sct", + description="Set column type", + resource_class=TableResource, + ), + Action( + name="drop-table", + abbr="dt", + description="Drop tables", + resource_class=TableResource, + ), + # Query-level actions (child-level) + Action( + name="view-query", + abbr="vq", + description="View named query results", + resource_class=QueryResource, + ), + Action( + name="update-query", + abbr="uq", + description="Update stored queries", + resource_class=QueryResource, + ), + Action( + name="delete-query", + abbr="dq", + description="Delete stored queries", + resource_class=QueryResource, + ), + ) diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py new file mode 100644 index 00000000..24493994 --- /dev/null +++ b/datasette/default_column_types.py @@ -0,0 +1,81 @@ +import json +import re + +import markupsafe + +from datasette import hookimpl +from datasette.column_types import ColumnType, SQLiteType + + +class UrlColumnType(ColumnType): + name = "url" + description = "URL" + sqlite_types = (SQLiteType.TEXT,) + + async def render_cell(self, value, column, table, database, datasette, request): + if not value or not isinstance(value, str): + return None + escaped = markupsafe.escape(value.strip()) + return markupsafe.Markup(f'{escaped}') + + async def validate(self, value, datasette): + if value is None or value == "": + return None + if not isinstance(value, str): + return "URL must be a string" + if not re.match(r"^https?://\S+$", value.strip()): + return "Invalid URL" + return None + + +class EmailColumnType(ColumnType): + name = "email" + description = "Email address" + sqlite_types = (SQLiteType.TEXT,) + + async def render_cell(self, value, column, table, database, datasette, request): + if not value or not isinstance(value, str): + return None + escaped = markupsafe.escape(value.strip()) + return markupsafe.Markup(f'{escaped}') + + async def validate(self, value, datasette): + if value is None or value == "": + return None + if not isinstance(value, str): + return "Email must be a string" + if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", value.strip()): + return "Invalid email address" + return None + + +class JsonColumnType(ColumnType): + name = "json" + description = "JSON data" + sqlite_types = (SQLiteType.TEXT,) + + async def render_cell(self, value, column, table, database, datasette, request): + if value is None: + return None + try: + parsed = json.loads(value) if isinstance(value, str) else value + formatted = json.dumps(parsed, indent=2) + escaped = markupsafe.escape(formatted) + return markupsafe.Markup(f"
{escaped}
") + except (json.JSONDecodeError, TypeError): + return None + + async def validate(self, value, datasette): + if value is None or value == "": + return None + if isinstance(value, str): + try: + json.loads(value) + except json.JSONDecodeError: + return "Invalid JSON" + return None + + +@hookimpl +def register_column_types(datasette): + return [UrlColumnType, EmailColumnType, JsonColumnType] diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py new file mode 100644 index 00000000..e0cb3cdf --- /dev/null +++ b/datasette/default_database_actions.py @@ -0,0 +1,24 @@ +from datasette import hookimpl +from datasette.resources import DatabaseResource + + +@hookimpl +def database_actions(datasette, actor, database, request): + async def inner(): + if not datasette.get_database(database).is_mutable: + return [] + if not await datasette.allowed( + action="execute-write-sql", + resource=DatabaseResource(database), + actor=actor, + ): + return [] + return [ + { + "href": datasette.urls.database(database) + "/-/execute-write", + "label": "Execute write SQL", + "description": "Run writable SQL with table permission checks.", + } + ] + + return inner diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py new file mode 100644 index 00000000..6127b2a6 --- /dev/null +++ b/datasette/default_debug_menu.py @@ -0,0 +1,75 @@ +from datasette import hookimpl +from datasette.jump import JumpSQL + +DEBUG_MENU_ITEMS = ( + ( + "/-/databases", + "Databases", + "List of databases known to this Datasette instance.", + ), + ( + "/-/plugins", + "Installed plugins", + "Review loaded plugins, their versions and their registered hooks.", + ), + ( + "/-/versions", + "Version info", + "Check the Python, SQLite and dependency versions used by this server.", + ), + ( + "/-/settings", + "Settings", + "Inspect the active Datasette settings and configuration values.", + ), + ( + "/-/permissions", + "Debug permissions", + "Test permission checks for actors, actions and resources.", + ), + ( + "/-/messages", + "Debug messages", + "Try out temporary flash messages shown to users.", + ), + ( + "/-/allow-debug", + "Debug allow rules", + "Explore how allow blocks match actors against permission rules.", + ), + ( + "/-/threads", + "Debug threads", + "Inspect worker threads and database tasks.", + ), + ( + "/-/actor", + "Debug actor", + "View the actor object for the current signed-in user.", + ), + ( + "/-/patterns", + "Pattern portfolio", + "Browse Datasette UI patterns.", + ), +) + + +@hookimpl +def jump_items_sql(datasette, actor, request): + async def inner(): + if not await datasette.allowed(action="debug-menu", actor=actor): + return [] + + return [ + JumpSQL.menu_item( + label=label, + url=datasette.urls.path(path), + description=description, + search_text=f"debug {label} {description}", + item_type="debug", + ) + for path, label, description in DEBUG_MENU_ITEMS + ] + + return inner diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py new file mode 100644 index 00000000..d215e7ec --- /dev/null +++ b/datasette/default_jump_items.py @@ -0,0 +1,82 @@ +from datasette import hookimpl +from datasette.jump import JumpSQL + + +@hookimpl +def jump_items_sql(datasette, actor, request): + async def inner(): + database_sql, database_params = await datasette.allowed_resources_sql( + action="view-database", actor=actor + ) + table_sql, table_params = await datasette.allowed_resources_sql( + action="view-table", actor=actor + ) + query_sql, query_params = await datasette.allowed_resources_sql( + action="view-query", actor=actor + ) + return [ + JumpSQL( + sql=f""" + WITH allowed_databases AS ( + {database_sql} + ) + SELECT + 'database' AS type, + parent AS label, + NULL AS description, + json_object( + 'method', 'database', + 'database', parent + ) AS url, + parent AS search_text, + NULL AS display_name + FROM allowed_databases + """, + params=database_params, + ), + JumpSQL( + sql=f""" + WITH allowed_tables AS ( + {table_sql} + ) + SELECT + CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, + allowed_tables.parent || ': ' || allowed_tables.child AS label, + NULL AS description, + json_object( + 'method', 'table', + 'database', allowed_tables.parent, + 'table', allowed_tables.child + ) AS url, + allowed_tables.parent || ' ' || allowed_tables.child AS search_text, + NULL AS display_name + FROM allowed_tables + LEFT JOIN catalog_views + ON catalog_views.database_name = allowed_tables.parent + AND catalog_views.view_name = allowed_tables.child + """, + params=table_params, + ), + JumpSQL( + sql=f""" + WITH allowed_queries AS ( + {query_sql} + ) + SELECT + 'query' AS type, + allowed_queries.parent || ': ' || allowed_queries.child AS label, + NULL AS description, + json_object( + 'method', 'query', + 'database', allowed_queries.parent, + 'query', allowed_queries.child + ) AS url, + allowed_queries.parent || ' ' || allowed_queries.child AS search_text, + NULL AS display_name + FROM allowed_queries + """, + params=query_params, + ), + ] + + return inner diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py deleted file mode 100644 index 22e6e46a..00000000 --- a/datasette/default_menu_links.py +++ /dev/null @@ -1,41 +0,0 @@ -from datasette import hookimpl - - -@hookimpl -def menu_links(datasette, actor): - async def inner(): - if not await datasette.permission_allowed(actor, "debug-menu"): - return [] - - return [ - {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, - { - "href": datasette.urls.path("/-/plugins"), - "label": "Installed plugins", - }, - { - "href": datasette.urls.path("/-/versions"), - "label": "Version info", - }, - { - "href": datasette.urls.path("/-/settings"), - "label": "Settings", - }, - { - "href": datasette.urls.path("/-/permissions"), - "label": "Debug permissions", - }, - { - "href": datasette.urls.path("/-/messages"), - "label": "Debug messages", - }, - { - "href": datasette.urls.path("/-/allow-debug"), - "label": "Debug allow rules", - }, - {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"}, - {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, - {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, - ] - - return inner diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py deleted file mode 100644 index a9534cab..00000000 --- a/datasette/default_permissions.py +++ /dev/null @@ -1,577 +0,0 @@ -from datasette import hookimpl, Permission -from datasette.utils.permissions import PluginSQL -from datasette.utils import actor_matches_allow -import itsdangerous -import time - - -@hookimpl -def register_permissions(): - return ( - Permission( - name="view-instance", - abbr="vi", - description="View Datasette instance", - takes_database=False, - takes_resource=False, - default=True, - ), - Permission( - name="view-database", - abbr="vd", - description="View database", - takes_database=True, - takes_resource=False, - default=True, - implies_can_view=True, - ), - Permission( - name="view-database-download", - abbr="vdd", - description="Download database file", - takes_database=True, - takes_resource=False, - default=True, - ), - Permission( - name="view-table", - abbr="vt", - description="View table", - takes_database=True, - takes_resource=True, - default=True, - implies_can_view=True, - ), - Permission( - name="view-query", - abbr="vq", - description="View named query results", - takes_database=True, - takes_resource=True, - default=True, - implies_can_view=True, - ), - Permission( - name="execute-sql", - abbr="es", - description="Execute read-only SQL queries", - takes_database=True, - takes_resource=False, - default=True, - implies_can_view=True, - ), - Permission( - name="permissions-debug", - abbr="pd", - description="Access permission debug tool", - takes_database=False, - takes_resource=False, - default=False, - ), - Permission( - name="debug-menu", - abbr="dm", - description="View debug menu items", - takes_database=False, - takes_resource=False, - default=False, - ), - Permission( - name="insert-row", - abbr="ir", - description="Insert rows", - takes_database=True, - takes_resource=True, - default=False, - ), - Permission( - name="delete-row", - abbr="dr", - description="Delete rows", - takes_database=True, - takes_resource=True, - default=False, - ), - Permission( - name="update-row", - abbr="ur", - description="Update rows", - takes_database=True, - takes_resource=True, - default=False, - ), - Permission( - name="create-table", - abbr="ct", - description="Create tables", - takes_database=True, - takes_resource=False, - default=False, - ), - Permission( - name="alter-table", - abbr="at", - description="Alter tables", - takes_database=True, - takes_resource=True, - default=False, - ), - Permission( - name="drop-table", - abbr="dt", - description="Drop tables", - takes_database=True, - takes_resource=True, - default=False, - ), - ) - - -@hookimpl(tryfirst=True, specname="permission_allowed") -def permission_allowed_default(datasette, actor, action, resource): - async def inner(): - # id=root gets some special permissions: - if action in ( - "permissions-debug", - "debug-menu", - "insert-row", - "create-table", - "alter-table", - "drop-table", - "delete-row", - "update-row", - ): - if actor and actor.get("id") == "root": - return True - - # Resolve view permissions in allow blocks in configuration - if action in ( - "view-instance", - "view-database", - "view-table", - "view-query", - "execute-sql", - ): - result = await _resolve_config_view_permissions( - datasette, actor, action, resource - ) - if result is not None: - return result - - # Resolve custom permissions: blocks in configuration - result = await _resolve_config_permissions_blocks( - datasette, actor, action, resource - ) - if result is not None: - return result - - # --setting default_allow_sql - if action == "execute-sql" and not datasette.setting("default_allow_sql"): - return False - - return inner - - -@hookimpl -async def permission_resources_sql(datasette, actor, action): - rules: list[PluginSQL] = [] - - config_rules = await _config_permission_rules(datasette, actor, action) - rules.extend(config_rules) - - default_allow_actions = { - "view-instance", - "view-database", - "view-table", - "execute-sql", - } - if action in default_allow_actions: - reason = f"default allow for {action}".replace("'", "''") - sql = ( - "SELECT NULL AS parent, NULL AS child, 1 AS allow, " f"'{reason}' AS reason" - ) - rules.append( - PluginSQL( - source="default_permissions", - sql=sql, - params={}, - ) - ) - - if not rules: - return None - if len(rules) == 1: - return rules[0] - return rules - - -async def _config_permission_rules(datasette, actor, action) -> list[PluginSQL]: - config = datasette.config or {} - - if actor is None: - actor_dict: dict | None = None - elif isinstance(actor, dict): - actor_dict = actor - else: - actor_lookup = await datasette.actors_from_ids([actor]) - actor_dict = actor_lookup.get(actor) or {"id": actor} - - def evaluate(allow_block): - if allow_block is None: - return None - return actor_matches_allow(actor_dict, allow_block) - - rows = [] - - def add_row(parent, child, result, scope): - if result is None: - return - rows.append( - ( - parent, - child, - bool(result), - f"config {'allow' if result else 'deny'} {scope}", - ) - ) - - root_perm = (config.get("permissions") or {}).get(action) - add_row(None, None, evaluate(root_perm), f"permissions for {action}") - - for db_name, db_config in (config.get("databases") or {}).items(): - db_perm = (db_config.get("permissions") or {}).get(action) - add_row( - db_name, None, evaluate(db_perm), f"permissions for {action} on {db_name}" - ) - - for table_name, table_config in (db_config.get("tables") or {}).items(): - table_perm = (table_config.get("permissions") or {}).get(action) - add_row( - db_name, - table_name, - evaluate(table_perm), - f"permissions for {action} on {db_name}/{table_name}", - ) - - if action == "view-table": - table_allow = (table_config or {}).get("allow") - add_row( - db_name, - table_name, - evaluate(table_allow), - f"allow for {action} on {db_name}/{table_name}", - ) - - for query_name, query_config in (db_config.get("queries") or {}).items(): - query_perm = (query_config.get("permissions") or {}).get(action) - add_row( - db_name, - query_name, - evaluate(query_perm), - f"permissions for {action} on {db_name}/{query_name}", - ) - if action == "view-query": - query_allow = (query_config or {}).get("allow") - add_row( - db_name, - query_name, - evaluate(query_allow), - f"allow for {action} on {db_name}/{query_name}", - ) - - if action == "view-database": - db_allow = db_config.get("allow") - add_row( - db_name, None, evaluate(db_allow), f"allow for {action} on {db_name}" - ) - - if action == "execute-sql": - db_allow_sql = db_config.get("allow_sql") - add_row(db_name, None, evaluate(db_allow_sql), f"allow_sql for {db_name}") - - if action == "view-instance": - allow_block = config.get("allow") - add_row(None, None, evaluate(allow_block), "allow for view-instance") - - if action == "view-table": - # Tables handled in loop - pass - - if action == "view-query": - # Queries handled in loop - pass - - if action == "execute-sql": - allow_sql = config.get("allow_sql") - add_row(None, None, evaluate(allow_sql), "allow_sql") - - if action == "view-database": - # already handled per-database - pass - - if not rows: - return [] - - parts = [] - params = {} - for idx, (parent, child, allow, reason) in enumerate(rows): - key = f"cfg_{idx}" - parts.append( - f"SELECT :{key}_parent AS parent, :{key}_child AS child, :{key}_allow AS allow, :{key}_reason AS reason" - ) - params[f"{key}_parent"] = parent - params[f"{key}_child"] = child - params[f"{key}_allow"] = 1 if allow else 0 - params[f"{key}_reason"] = reason - - sql = "\nUNION ALL\n".join(parts) - print(sql, params) - return [PluginSQL(source="config_permissions", sql=sql, params=params)] - - -async def _resolve_config_permissions_blocks(datasette, actor, action, resource): - # Check custom permissions: blocks - config = datasette.config or {} - root_block = (config.get("permissions", None) or {}).get(action) - if root_block: - root_result = actor_matches_allow(actor, root_block) - if root_result is not None: - return root_result - # Now try database-specific blocks - if not resource: - return None - if isinstance(resource, str): - database = resource - else: - database = resource[0] - database_block = ( - (config.get("databases", {}).get(database, {}).get("permissions", None)) or {} - ).get(action) - if database_block: - database_result = actor_matches_allow(actor, database_block) - if database_result is not None: - return database_result - # Finally try table/query specific blocks - if not isinstance(resource, tuple): - return None - database, table_or_query = resource - table_block = ( - ( - config.get("databases", {}) - .get(database, {}) - .get("tables", {}) - .get(table_or_query, {}) - .get("permissions", None) - ) - or {} - ).get(action) - if table_block: - table_result = actor_matches_allow(actor, table_block) - if table_result is not None: - return table_result - # Finally the canned queries - query_block = ( - ( - config.get("databases", {}) - .get(database, {}) - .get("queries", {}) - .get(table_or_query, {}) - .get("permissions", None) - ) - or {} - ).get(action) - if query_block: - query_result = actor_matches_allow(actor, query_block) - if query_result is not None: - return query_result - return None - - -async def _resolve_config_view_permissions(datasette, actor, action, resource): - config = datasette.config or {} - if action == "view-instance": - allow = config.get("allow") - if allow is not None: - return actor_matches_allow(actor, allow) - elif action == "view-database": - database_allow = ((config.get("databases") or {}).get(resource) or {}).get( - "allow" - ) - if database_allow is None: - return None - return actor_matches_allow(actor, database_allow) - elif action == "view-table": - database, table = resource - tables = ((config.get("databases") or {}).get(database) or {}).get( - "tables" - ) or {} - table_allow = (tables.get(table) or {}).get("allow") - if table_allow is None: - return None - return actor_matches_allow(actor, table_allow) - elif action == "view-query": - # Check if this query has a "allow" block in config - database, query_name = resource - query = await datasette.get_canned_query(database, query_name, actor) - assert query is not None - allow = query.get("allow") - if allow is None: - return None - return actor_matches_allow(actor, allow) - elif action == "execute-sql": - # Use allow_sql block from database block, or from top-level - database_allow_sql = ((config.get("databases") or {}).get(resource) or {}).get( - "allow_sql" - ) - if database_allow_sql is None: - database_allow_sql = config.get("allow_sql") - if database_allow_sql is None: - return None - return actor_matches_allow(actor, database_allow_sql) - - -def restrictions_allow_action( - datasette: "Datasette", - restrictions: dict, - action: str, - resource: str | tuple[str, str], -): - "Do these restrictions allow the requested action against the requested resource?" - if action == "view-instance": - # Special case for view-instance: it's allowed if the restrictions include any - # permissions that have the implies_can_view=True flag set - all_rules = restrictions.get("a") or [] - for database_rules in (restrictions.get("d") or {}).values(): - all_rules += database_rules - for database_resource_rules in (restrictions.get("r") or {}).values(): - for resource_rules in database_resource_rules.values(): - all_rules += resource_rules - permissions = [datasette.get_permission(action) for action in all_rules] - if any(p for p in permissions if p.implies_can_view): - return True - - if action == "view-database": - # Special case for view-database: it's allowed if the restrictions include any - # permissions that have the implies_can_view=True flag set AND takes_database - all_rules = restrictions.get("a") or [] - database_rules = list((restrictions.get("d") or {}).get(resource) or []) - all_rules += database_rules - resource_rules = ((restrictions.get("r") or {}).get(resource) or {}).values() - for resource_rules in (restrictions.get("r") or {}).values(): - for table_rules in resource_rules.values(): - all_rules += table_rules - permissions = [datasette.get_permission(action) for action in all_rules] - if any(p for p in permissions if p.implies_can_view and p.takes_database): - return True - - # Does this action have an abbreviation? - to_check = {action} - permission = datasette.permissions.get(action) - if permission and permission.abbr: - to_check.add(permission.abbr) - - # If restrictions is defined then we use those to further restrict the actor - # Crucially, we only use this to say NO (return False) - we never - # use it to return YES (True) because that might over-ride other - # restrictions placed on this actor - all_allowed = restrictions.get("a") - if all_allowed is not None: - assert isinstance(all_allowed, list) - if to_check.intersection(all_allowed): - return True - # How about for the current database? - if resource: - if isinstance(resource, str): - database_name = resource - else: - database_name = resource[0] - database_allowed = restrictions.get("d", {}).get(database_name) - if database_allowed is not None: - assert isinstance(database_allowed, list) - if to_check.intersection(database_allowed): - return True - # Or the current table? That's any time the resource is (database, table) - if resource is not None and not isinstance(resource, str) and len(resource) == 2: - database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - # TODO: What should this do for canned queries? - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True - - # This action is not specifically allowed, so reject it - return False - - -@hookimpl(specname="permission_allowed") -def permission_allowed_actor_restrictions(datasette, actor, action, resource): - if actor is None: - return None - if "_r" not in actor: - # No restrictions, so we have no opinion - return None - _r = actor.get("_r") - if restrictions_allow_action(datasette, _r, action, resource): - # Return None because we do not have an opinion here - return None - else: - # Block this permission check - return False - - -@hookimpl -def actor_from_request(datasette, request): - prefix = "dstok_" - if not datasette.setting("allow_signed_tokens"): - return None - max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") - authorization = request.headers.get("authorization") - if not authorization: - return None - if not authorization.startswith("Bearer "): - return None - token = authorization[len("Bearer ") :] - if not token.startswith(prefix): - return None - token = token[len(prefix) :] - try: - decoded = datasette.unsign(token, namespace="token") - except itsdangerous.BadSignature: - return None - if "t" not in decoded: - # Missing timestamp - return None - created = decoded["t"] - if not isinstance(created, int): - # Invalid timestamp - return None - duration = decoded.get("d") - if duration is not None and not isinstance(duration, int): - # Invalid duration - return None - if (duration is None and max_signed_tokens_ttl) or ( - duration is not None - and max_signed_tokens_ttl - and duration > max_signed_tokens_ttl - ): - duration = max_signed_tokens_ttl - if duration: - if time.time() - created > duration: - # Expired - return None - actor = {"id": decoded["a"], "token": "dstok"} - if "_r" in decoded: - actor["_r"] = decoded["_r"] - if duration: - actor["token_expires"] = created + duration - return actor - - -@hookimpl -def skip_csrf(scope): - # Skip CSRF check for requests with content-type: application/json - if scope["type"] == "http": - headers = scope.get("headers") or {} - if dict(headers).get(b"content-type") == b"application/json": - return True diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py new file mode 100644 index 00000000..6cd46f04 --- /dev/null +++ b/datasette/default_permissions/__init__.py @@ -0,0 +1,34 @@ +""" +Default permission implementations for Datasette. + +This module provides the built-in permission checking logic through implementations +of the permission_resources_sql hook. The hooks are organized by their purpose: + +1. Actor Restrictions - Enforces _r allowlists embedded in actor tokens +2. Root User - Grants full access when --root flag is used +3. Config Rules - Applies permissions from datasette.yaml +4. Default Settings - Enforces default_allow_sql and default view permissions + +IMPORTANT: These hooks return PermissionSQL objects that are combined using SQL +UNION/INTERSECT operations. The order of evaluation is: + - restriction_sql fields are INTERSECTed (all must match) + - Regular sql fields are UNIONed and evaluated with cascading priority +""" + +from __future__ import annotations + +# Re-export all hooks and public utilities +from .restrictions import ( + actor_restrictions_sql as actor_restrictions_sql, + restrictions_allow_action as restrictions_allow_action, + ActorRestrictions as ActorRestrictions, +) +from .root import root_user_permissions_sql as root_user_permissions_sql +from .config import config_permissions_sql as config_permissions_sql +from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute + default_allow_sql_check as default_allow_sql_check, + default_action_permissions_sql as default_action_permissions_sql, + default_query_permissions_sql as default_query_permissions_sql, + DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, +) diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py new file mode 100644 index 00000000..aab87c1c --- /dev/null +++ b/datasette/default_permissions/config.py @@ -0,0 +1,442 @@ +""" +Config-based permission handling for Datasette. + +Applies permission rules from datasette.yaml configuration. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette import hookimpl +from datasette.permissions import PermissionSQL +from datasette.utils import actor_matches_allow + +from .helpers import PermissionRowCollector, get_action_name_variants + + +class ConfigPermissionProcessor: + """ + Processes permission rules from datasette.yaml configuration. + + Configuration structure: + + permissions: # Root-level permissions block + view-instance: + id: admin + + databases: + mydb: + permissions: # Database-level permissions + view-database: + id: admin + allow: # Database-level allow block (for view-*) + id: viewer + allow_sql: # execute-sql allow block + id: analyst + tables: + users: + permissions: # Table-level permissions + view-table: + id: admin + allow: # Table-level allow block + id: viewer + queries: + my_query: + permissions: # Query-level permissions + view-query: + id: admin + allow: # Query-level allow block + id: viewer + """ + + def __init__( + self, + datasette: "Datasette", + actor: Optional[dict], + action: str, + ): + self.datasette = datasette + self.actor = actor + self.action = action + self.config = datasette.config or {} + self.collector = PermissionRowCollector(prefix="cfg") + + # Pre-compute action variants + self.action_checks = get_action_name_variants(datasette, action) + self.action_obj = datasette.actions.get(action) + + # Parse restrictions if present + self.has_restrictions = actor and "_r" in actor if actor else False + self.restrictions = actor.get("_r", {}) if actor else {} + + # Pre-compute restriction info for efficiency + self.restricted_databases: Set[str] = set() + self.restricted_tables: Set[Tuple[str, str]] = set() + + if self.has_restrictions: + self.restricted_databases = { + db_name + for db_name, db_actions in (self.restrictions.get("d") or {}).items() + if self.action_checks.intersection(db_actions) + } + self.restricted_tables = { + (db_name, table_name) + for db_name, tables in (self.restrictions.get("r") or {}).items() + for table_name, table_actions in tables.items() + if self.action_checks.intersection(table_actions) + } + # Tables implicitly reference their parent databases + self.restricted_databases.update(db for db, _ in self.restricted_tables) + + def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: + """Evaluate an allow block against the current actor.""" + if allow_block is None: + return None + return actor_matches_allow(self.actor, allow_block) + + def is_in_restriction_allowlist( + self, + parent: Optional[str], + child: Optional[str], + ) -> bool: + """Check if resource is allowed by actor restrictions.""" + if not self.has_restrictions: + return True # No restrictions, all resources allowed + + # Check global allowlist + if self.action_checks.intersection(self.restrictions.get("a", [])): + return True + + # Check database-level allowlist + if parent and self.action_checks.intersection( + self.restrictions.get("d", {}).get(parent, []) + ): + return True + + # Check table-level allowlist + if parent: + table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) + if child: + table_actions = table_restrictions.get(child, []) + if self.action_checks.intersection(table_actions): + return True + else: + # Parent query should proceed if any child in this database is allowlisted + for table_actions in table_restrictions.values(): + if self.action_checks.intersection(table_actions): + return True + + # Parent/child both None: include if any restrictions exist for this action + if parent is None and child is None: + if self.action_checks.intersection(self.restrictions.get("a", [])): + return True + if self.restricted_databases: + return True + if self.restricted_tables: + return True + + return False + + def add_permissions_rule( + self, + parent: Optional[str], + child: Optional[str], + permissions_block: Optional[dict], + scope_desc: str, + ) -> None: + """Add a rule from a permissions:{action} block.""" + if permissions_block is None: + return + + action_allow_block = permissions_block.get(self.action) + result = self.evaluate_allow_block(action_allow_block) + + self.collector.add( + parent=parent, + child=child, + allow=result, + reason=f"config {'allow' if result else 'deny'} {scope_desc}", + if_not_none=True, + ) + + def add_allow_block_rule( + self, + parent: Optional[str], + child: Optional[str], + allow_block: Any, + scope_desc: str, + ) -> None: + """ + Add rules from an allow:{} block. + + For allow blocks, if the block exists but doesn't match the actor, + this is treated as a deny. We also handle the restriction-gate logic. + """ + if allow_block is None: + return + + # Skip if resource is not in restriction allowlist + if not self.is_in_restriction_allowlist(parent, child): + return + + result = self.evaluate_allow_block(allow_block) + bool_result = bool(result) + + self.collector.add( + parent, + child, + bool_result, + f"config {'allow' if result else 'deny'} {scope_desc}", + ) + + # Handle restriction-gate: add explicit denies for restricted resources + self._add_restriction_gate_denies(parent, child, bool_result, scope_desc) + + def _add_restriction_gate_denies( + self, + parent: Optional[str], + child: Optional[str], + is_allowed: bool, + scope_desc: str, + ) -> None: + """ + When a config rule denies at a higher level, add explicit denies + for restricted resources to prevent child-level allows from + incorrectly granting access. + """ + if is_allowed or child is not None or not self.has_restrictions: + return + + if not self.action_obj: + return + + reason = f"config deny {scope_desc} (restriction gate)" + + if parent is None: + # Root-level deny: add denies for all restricted resources + if self.action_obj.takes_parent: + for db_name in self.restricted_databases: + self.collector.add(db_name, None, False, reason) + if self.action_obj.takes_child: + for db_name, table_name in self.restricted_tables: + self.collector.add(db_name, table_name, False, reason) + else: + # Database-level deny: add denies for tables in that database + if self.action_obj.takes_child: + for db_name, table_name in self.restricted_tables: + if db_name == parent: + self.collector.add(db_name, table_name, False, reason) + + def process(self) -> Optional[PermissionSQL]: + """Process all config rules and return combined PermissionSQL.""" + self._process_root_permissions() + self._process_databases() + self._process_root_allow_blocks() + + return self.collector.to_permission_sql() + + def _process_root_permissions(self) -> None: + """Process root-level permissions block.""" + root_perms = self.config.get("permissions") or {} + self.add_permissions_rule( + None, + None, + root_perms, + f"permissions for {self.action}", + ) + + def _process_databases(self) -> None: + """Process database-level and nested configurations.""" + databases = self.config.get("databases") or {} + + for db_name, db_config in databases.items(): + self._process_database(db_name, db_config or {}) + + def _process_database(self, db_name: str, db_config: dict) -> None: + """Process a single database's configuration.""" + # Database-level permissions block + db_perms = db_config.get("permissions") or {} + self.add_permissions_rule( + db_name, + None, + db_perms, + f"permissions for {self.action} on {db_name}", + ) + + # Process tables + for table_name, table_config in (db_config.get("tables") or {}).items(): + self._process_table(db_name, table_name, table_config or {}) + + # Process queries + for query_name, query_config in (db_config.get("queries") or {}).items(): + self._process_query(db_name, query_name, query_config) + + # Database-level allow blocks + self._process_database_allow_blocks(db_name, db_config) + + def _process_table( + self, + db_name: str, + table_name: str, + table_config: dict, + ) -> None: + """Process a single table's configuration.""" + # Table-level permissions block + table_perms = table_config.get("permissions") or {} + self.add_permissions_rule( + db_name, + table_name, + table_perms, + f"permissions for {self.action} on {db_name}/{table_name}", + ) + + # Table-level allow block (for view-table) + if self.action == "view-table": + self.add_allow_block_rule( + db_name, + table_name, + table_config.get("allow"), + f"allow for {self.action} on {db_name}/{table_name}", + ) + + def _process_query( + self, + db_name: str, + query_name: str, + query_config: Any, + ) -> None: + """Process a single query's configuration.""" + # Query config can be a string (just SQL) or dict + if not isinstance(query_config, dict): + return + + # Query-level permissions block + query_perms = query_config.get("permissions") or {} + self.add_permissions_rule( + db_name, + query_name, + query_perms, + f"permissions for {self.action} on {db_name}/{query_name}", + ) + + # Query-level allow block (for view-query) + if self.action == "view-query": + self.add_allow_block_rule( + db_name, + query_name, + query_config.get("allow"), + f"allow for {self.action} on {db_name}/{query_name}", + ) + + def _process_database_allow_blocks( + self, + db_name: str, + db_config: dict, + ) -> None: + """Process database-level allow/allow_sql blocks.""" + # view-database allow block + if self.action == "view-database": + self.add_allow_block_rule( + db_name, + None, + db_config.get("allow"), + f"allow for {self.action} on {db_name}", + ) + + # execute-sql allow_sql block + if self.action == "execute-sql": + self.add_allow_block_rule( + db_name, + None, + db_config.get("allow_sql"), + f"allow_sql for {db_name}", + ) + + # view-table uses database-level allow for inheritance + if self.action == "view-table": + self.add_allow_block_rule( + db_name, + None, + db_config.get("allow"), + f"allow for {self.action} on {db_name}", + ) + + # view-query uses database-level allow for inheritance + if self.action == "view-query": + self.add_allow_block_rule( + db_name, + None, + db_config.get("allow"), + f"allow for {self.action} on {db_name}", + ) + + def _process_root_allow_blocks(self) -> None: + """Process root-level allow/allow_sql blocks.""" + root_allow = self.config.get("allow") + + if self.action == "view-instance": + self.add_allow_block_rule( + None, + None, + root_allow, + "allow for view-instance", + ) + + if self.action == "view-database": + self.add_allow_block_rule( + None, + None, + root_allow, + "allow for view-database", + ) + + if self.action == "view-table": + self.add_allow_block_rule( + None, + None, + root_allow, + "allow for view-table", + ) + + if self.action == "view-query": + self.add_allow_block_rule( + None, + None, + root_allow, + "allow for view-query", + ) + + if self.action == "execute-sql": + self.add_allow_block_rule( + None, + None, + self.config.get("allow_sql"), + "allow_sql", + ) + + +@hookimpl(specname="permission_resources_sql") +async def config_permissions_sql( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[List[PermissionSQL]]: + """ + Apply permission rules from datasette.yaml configuration. + + This processes: + - permissions: blocks at root, database, table, and query levels + - allow: blocks for view-* actions + - allow_sql: blocks for execute-sql action + """ + processor = ConfigPermissionProcessor(datasette, actor, action) + result = processor.process() + + if result is None: + return [] + + return [result] diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py new file mode 100644 index 00000000..5bc74425 --- /dev/null +++ b/datasette/default_permissions/defaults.py @@ -0,0 +1,114 @@ +""" +Default permission settings for Datasette. + +Provides default allow rules for standard view/execute actions. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + +# Actions that are allowed by default (unless --default-deny is used) +DEFAULT_ALLOW_ACTIONS = frozenset( + { + "view-instance", + "view-database", + "view-database-download", + "view-table", + "view-query", + "execute-sql", + } +) + + +@hookimpl(specname="permission_resources_sql") +async def default_allow_sql_check( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[PermissionSQL]: + """ + Enforce the default_allow_sql setting. + + When default_allow_sql is false (the default), execute-sql is denied + unless explicitly allowed by config or other rules. + """ + if action == "execute-sql": + if not datasette.setting("default_allow_sql"): + return PermissionSQL.deny(reason="default_allow_sql is false") + + return None + + +@hookimpl(specname="permission_resources_sql") +async def default_action_permissions_sql( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[PermissionSQL]: + """ + Provide default allow rules for standard view/execute actions. + + These defaults are skipped when datasette is started with --default-deny. + The restriction_sql mechanism (from actor_restrictions_sql) will still + filter these results if the actor has restrictions. + """ + if datasette.default_deny: + return None + + if action in DEFAULT_ALLOW_ACTIONS: + reason = f"default allow for {action}".replace("'", "''") + return PermissionSQL.allow(reason=reason) + + return None + + +@hookimpl(specname="permission_resources_sql") +async def default_query_permissions_sql( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[PermissionSQL]: + actor_id = actor.get("id") if isinstance(actor, dict) else None + + if action not in {"view-query", "update-query", "delete-query"}: + return None + + params = {"query_owner_id": actor_id} + rule_sqls = [] + if actor_id is not None: + if action in {"update-query", "delete-query"}: + # Query owner can update/delete query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE source = 'user' + AND owner_id = :query_owner_id + """) + else: + # Query owner can view-query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE owner_id = :query_owner_id + """) + + # restriction_sql enforces private queries ONLY visible/mutable by owner + return PermissionSQL( + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql=""" + SELECT database_name AS parent, name AS child + FROM queries + WHERE is_private = 0 + OR owner_id = :query_owner_id + """, + params=params, + ) diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py new file mode 100644 index 00000000..47e03569 --- /dev/null +++ b/datasette/default_permissions/helpers.py @@ -0,0 +1,85 @@ +""" +Shared helper utilities for default permission implementations. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Set + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette.permissions import PermissionSQL + + +def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: + """ + Get all name variants for an action (full name and abbreviation). + + Example: + get_action_name_variants(ds, "view-table") -> {"view-table", "vt"} + """ + variants = {action} + action_obj = datasette.actions.get(action) + if action_obj and action_obj.abbr: + variants.add(action_obj.abbr) + return variants + + +def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: + """Check if an action (or its abbreviation) is in a list.""" + return bool(get_action_name_variants(datasette, action).intersection(action_list)) + + +@dataclass +class PermissionRow: + """A single permission rule row.""" + + parent: Optional[str] + child: Optional[str] + allow: bool + reason: str + + +class PermissionRowCollector: + """Collects permission rows and converts them to PermissionSQL.""" + + def __init__(self, prefix: str = "row"): + self.rows: List[PermissionRow] = [] + self.prefix = prefix + + def add( + self, + parent: Optional[str], + child: Optional[str], + allow: Optional[bool], + reason: str, + if_not_none: bool = False, + ) -> None: + """Add a permission row. If if_not_none=True, only add if allow is not None.""" + if if_not_none and allow is None: + return + self.rows.append(PermissionRow(parent, child, allow, reason)) + + def to_permission_sql(self) -> Optional[PermissionSQL]: + """Convert collected rows to a PermissionSQL object.""" + if not self.rows: + return None + + parts = [] + params = {} + + for idx, row in enumerate(self.rows): + key = f"{self.prefix}_{idx}" + parts.append( + f"SELECT :{key}_parent AS parent, :{key}_child AS child, " + f":{key}_allow AS allow, :{key}_reason AS reason" + ) + params[f"{key}_parent"] = row.parent + params[f"{key}_child"] = row.child + params[f"{key}_allow"] = 1 if row.allow else 0 + params[f"{key}_reason"] = row.reason + + sql = "\nUNION ALL\n".join(parts) + return PermissionSQL(sql=sql, params=params) diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py new file mode 100644 index 00000000..a22cd7e5 --- /dev/null +++ b/datasette/default_permissions/restrictions.py @@ -0,0 +1,195 @@ +""" +Actor restriction handling for Datasette permissions. + +This module handles the _r (restrictions) key in actor dictionaries, which +contains allowlists of resources the actor can access. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Set, Tuple + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + +from .helpers import action_in_list, get_action_name_variants + + +@dataclass +class ActorRestrictions: + """Parsed actor restrictions from the _r key.""" + + global_actions: List[str] # _r.a - globally allowed actions + database_actions: dict # _r.d - {db_name: [actions]} + table_actions: dict # _r.r - {db_name: {table: [actions]}} + + @classmethod + def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: + """Parse restrictions from actor dict. Returns None if no restrictions.""" + if not actor: + return None + assert isinstance(actor, dict), "actor must be a dictionary" + + restrictions = actor.get("_r") + if restrictions is None: + return None + + return cls( + global_actions=restrictions.get("a", []), + database_actions=restrictions.get("d", {}), + table_actions=restrictions.get("r", {}), + ) + + def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: + """Check if action is in the global allowlist.""" + return action_in_list(datasette, action, self.global_actions) + + def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: + """Get database names where this action is allowed.""" + allowed = set() + for db_name, db_actions in self.database_actions.items(): + if action_in_list(datasette, action, db_actions): + allowed.add(db_name) + return allowed + + def get_allowed_tables( + self, datasette: "Datasette", action: str + ) -> Set[Tuple[str, str]]: + """Get (database, table) pairs where this action is allowed.""" + allowed = set() + for db_name, tables in self.table_actions.items(): + for table_name, table_actions in tables.items(): + if action_in_list(datasette, action, table_actions): + allowed.add((db_name, table_name)) + return allowed + + +@hookimpl(specname="permission_resources_sql") +async def actor_restrictions_sql( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[List[PermissionSQL]]: + """ + Handle actor restriction-based permission rules. + + When an actor has an "_r" key, it contains an allowlist of resources they + can access. This function returns restriction_sql that filters the final + results to only include resources in that allowlist. + + The _r structure: + { + "a": ["vi", "pd"], # Global actions allowed + "d": {"mydb": ["vt", "es"]}, # Database-level actions + "r": {"mydb": {"users": ["vt"]}} # Table-level actions + } + """ + if not actor: + return None + + restrictions = ActorRestrictions.from_actor(actor) + + if restrictions is None: + # No restrictions - all resources allowed + return [] + + # If globally allowed, no filtering needed + if restrictions.is_action_globally_allowed(datasette, action): + return [] + + # Build restriction SQL + allowed_dbs = restrictions.get_allowed_databases(datasette, action) + allowed_tables = restrictions.get_allowed_tables(datasette, action) + + # If nothing is allowed for this action, return empty-set restriction + if not allowed_dbs and not allowed_tables: + return [ + PermissionSQL( + params={"deny": f"actor restrictions: {action} not in allowlist"}, + restriction_sql="SELECT NULL AS parent, NULL AS child WHERE 0", + ) + ] + + # Build UNION of allowed resources + selects = [] + params = {} + counter = 0 + + # Database-level entries (parent, NULL) - allows all children + for db_name in allowed_dbs: + key = f"restr_{counter}" + counter += 1 + selects.append(f"SELECT :{key}_parent AS parent, NULL AS child") + params[f"{key}_parent"] = db_name + + # Table-level entries (parent, child) + for db_name, table_name in allowed_tables: + key = f"restr_{counter}" + counter += 1 + selects.append(f"SELECT :{key}_parent AS parent, :{key}_child AS child") + params[f"{key}_parent"] = db_name + params[f"{key}_child"] = table_name + + restriction_sql = "\nUNION ALL\n".join(selects) + + return [PermissionSQL(params=params, restriction_sql=restriction_sql)] + + +def restrictions_allow_action( + datasette: "Datasette", + restrictions: dict, + action: str, + resource: Optional[str | Tuple[str, str]], +) -> bool: + """ + Check if restrictions allow the requested action on the requested resource. + + This is a synchronous utility function for use by other code that needs + to quickly check restriction allowlists. + + Args: + datasette: The Datasette instance + restrictions: The _r dict from an actor + action: The action name to check + resource: None for global, str for database, (db, table) tuple for table + + Returns: + True if allowed, False if denied + """ + # Does this action have an abbreviation? + to_check = get_action_name_variants(datasette, action) + + # Check global level (any resource) + all_allowed = restrictions.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if to_check.intersection(all_allowed): + return True + + # Check database level + if resource: + if isinstance(resource, str): + database_name = resource + else: + database_name = resource[0] + database_allowed = restrictions.get("d", {}).get(database_name) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if to_check.intersection(database_allowed): + return True + + # Check table/resource level + if resource is not None and not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True + + # This action is not explicitly allowed, so reject it + return False diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py new file mode 100644 index 00000000..4931f7ff --- /dev/null +++ b/datasette/default_permissions/root.py @@ -0,0 +1,29 @@ +""" +Root user permission handling for Datasette. + +Grants full permissions to the root user when --root flag is used. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + + +@hookimpl(specname="permission_resources_sql") +async def root_user_permissions_sql( + datasette: "Datasette", + actor: Optional[dict], +) -> Optional[PermissionSQL]: + """ + Grant root user full permissions when --root flag is used. + """ + if not datasette.root_enabled: + return None + if actor is not None and actor.get("id") == "root": + return PermissionSQL.allow(reason="root user") diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py new file mode 100644 index 00000000..7a359dc6 --- /dev/null +++ b/datasette/default_permissions/tokens.py @@ -0,0 +1,40 @@ +""" +Token authentication for Datasette. + +Registers the default SignedTokenHandler and delegates token verification +to datasette.verify_token() so all registered handlers are tried. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette import hookimpl +from datasette.tokens import SignedTokenHandler + + +@hookimpl +def register_token_handler(datasette: "Datasette"): + """Register the default signed token handler.""" + return SignedTokenHandler() + + +@hookimpl(specname="actor_from_request") +async def actor_from_signed_api_token( + datasette: "Datasette", request +) -> Optional[dict]: + """ + Authenticate requests using API tokens by delegating to all registered + token handlers via datasette.verify_token(). + """ + authorization = request.headers.get("authorization") + if not authorization: + return None + if not authorization.startswith("Bearer "): + return None + + token = authorization[len("Bearer ") :] + return await datasette.verify_token(token) diff --git a/datasette/events.py b/datasette/events.py index ae90972d..e8786da9 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -2,7 +2,6 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field from datasette.hookspecs import hookimpl from datetime import datetime, timezone -from typing import Optional @dataclass @@ -14,7 +13,7 @@ class Event(ABC): created: datetime = field( init=False, default_factory=lambda: datetime.now(timezone.utc) ) - actor: Optional[dict] + actor: dict | None def properties(self): properties = asdict(self) @@ -63,7 +62,7 @@ class CreateTokenEvent(Event): """ name = "create-token" - expires_after: Optional[int] + expires_after: int | None restrict_all: list restrict_database: dict restrict_resource: dict @@ -200,6 +199,27 @@ class UpdateRowEvent(Event): pks: list +@dataclass +class RenameTableEvent(Event): + """ + Event name: ``rename-table`` + + A table has been renamed. + + :ivar database: The name of the database containing the renamed table. + :type database: str + :ivar old_table: The previous name of the table. + :type old_table: str + :ivar new_table: The new name of the table. + :type new_table: str + """ + + name = "rename-table" + database: str + old_table: str + new_table: str + + @dataclass class DeleteRowEvent(Event): """ @@ -220,6 +240,42 @@ class DeleteRowEvent(Event): pks: list +@hookimpl +def write_wrapper(datasette, database, request, transaction): + def wrapper(conn, track_event): + # Snapshot rootpage -> name before the write + before = { + row[1]: row[0] + for row in conn.execute( + "select name, rootpage from sqlite_master" + " where type='table' and rootpage != 0" + ).fetchall() + } + yield + # Snapshot rootpage -> name after the write + after = { + row[1]: row[0] + for row in conn.execute( + "select name, rootpage from sqlite_master" + " where type='table' and rootpage != 0" + ).fetchall() + } + # Detect renames: same rootpage, different name + for rootpage, old_name in before.items(): + new_name = after.get(rootpage) + if new_name and new_name != old_name: + track_event( + RenameTableEvent( + actor=request.actor if request else None, + database=database, + old_table=old_name, + new_table=new_name, + ) + ) + + return wrapper + + @hookimpl def register_events(): return [ @@ -228,6 +284,7 @@ def register_events(): CreateTableEvent, CreateTokenEvent, AlterTableEvent, + RenameTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/facets.py b/datasette/facets.py index dd149424..abe0605e 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -83,7 +83,7 @@ class Facet: self.ds = ds self.request = request self.database = database - # For foreign key expansion. Can be None for e.g. canned SQL queries: + # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] @@ -233,9 +233,7 @@ class ColumnFacet(Facet): ) where {col} is not null group by {col} order by count desc, value limit {limit} - """.format( - col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 - ) + """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1) try: facet_rows_results = await self.ds.execute( self.database, @@ -482,9 +480,7 @@ class DateFacet(Facet): select date({column}) from ( select * from ({sql}) limit 100 ) where {column} glob "????-??-*" - """.format( - column=escape_sqlite(column), sql=self.sql - ) + """.format(column=escape_sqlite(column), sql=self.sql) try: results = await self.ds.execute( self.database, @@ -530,9 +526,7 @@ class DateFacet(Facet): ) where date({col}) is not null group by date({col}) order by count desc, value limit {limit} - """.format( - col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 - ) + """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1) try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 67d4170b..95cc5f37 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,8 +1,8 @@ from datasette import hookimpl +from datasette.resources import DatabaseResource from datasette.views.base import DatasetteError from datasette.utils.asgi import BadRequest import json -import numbers from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -13,11 +13,10 @@ def where_filters(request, database, datasette): where_clauses = [] extra_wheres_for_ui = [] if "_where" in request.args: - if not await datasette.permission_allowed( - request.actor, - "execute-sql", - resource=database, - default=True, + if not await datasette.allowed( + action="execute-sql", + resource=DatabaseResource(database=database), + actor=request.actor, ): raise DatasetteError("_where= is not allowed", status=403) else: diff --git a/datasette/fixtures.py b/datasette/fixtures.py new file mode 100644 index 00000000..7c85e16a --- /dev/null +++ b/datasette/fixtures.py @@ -0,0 +1,415 @@ +from datasette.utils.sqlite import sqlite3 +from datasette.utils import documented +import itertools +import random +import string + +__all__ = [ + "EXTRA_DATABASE_SQL", + "TABLES", + "TABLE_PARAMETERIZED_SQL", + "generate_compound_rows", + "generate_sortable_rows", + "populate_extra_database", + "populate_fixture_database", + "write_extra_database", + "write_fixture_database", +] + + +def generate_compound_rows(num): + """Generate rows for the compound_three_primary_keys fixture table.""" + for a, b, c in itertools.islice( + itertools.product(string.ascii_lowercase, repeat=3), num + ): + yield a, b, c, f"{a}-{b}-{c}" + + +def generate_sortable_rows(num): + """Generate rows for the sortable fixture table.""" + rand = random.Random(42) + for a, b in itertools.islice( + itertools.product(string.ascii_lowercase, repeat=2), num + ): + yield { + "pk1": a, + "pk2": b, + "content": f"{a}-{b}", + "sortable": rand.randint(-100, 100), + "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), + "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), + "text": rand.choice(["$null", "$blah"]), + } + + +TABLES = ( + """ +CREATE TABLE simple_primary_key ( + id integer primary key, + content text +); + +CREATE TABLE primary_key_multiple_columns ( + id varchar(30) primary key, + content text, + content2 text +); + +CREATE TABLE primary_key_multiple_columns_explicit_label ( + id varchar(30) primary key, + content text, + content2 text +); + +CREATE TABLE compound_primary_key ( + pk1 varchar(30), + pk2 varchar(30), + content text, + PRIMARY KEY (pk1, pk2) +); + +INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); +INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); +INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); + +CREATE TABLE compound_three_primary_keys ( + pk1 varchar(30), + pk2 varchar(30), + pk3 varchar(30), + content text, + PRIMARY KEY (pk1, pk2, pk3) +); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); + +CREATE TABLE foreign_key_references ( + pk varchar(30) primary key, + foreign_key_with_label integer, + foreign_key_with_blank_label integer, + foreign_key_with_no_label varchar(30), + foreign_key_compound_pk1 varchar(30), + foreign_key_compound_pk2 varchar(30), + FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) + FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) +); + +CREATE TABLE sortable ( + pk1 varchar(30), + pk2 varchar(30), + content text, + sortable integer, + sortable_with_nulls real, + sortable_with_nulls_2 real, + text text, + PRIMARY KEY (pk1, pk2) +); + +CREATE TABLE no_primary_key ( + content text, + a text, + b text, + c text +); + +CREATE TABLE [123_starts_with_digits] ( + content text +); + +CREATE VIEW paginated_view AS + SELECT + content, + '- ' || content || ' -' AS content_extra + FROM no_primary_key; + +CREATE TABLE "Table With Space In Name" ( + pk varchar(30) primary key, + content text +); + +CREATE TABLE "table/with/slashes.csv" ( + pk varchar(30) primary key, + content text +); + +CREATE TABLE "complex_foreign_keys" ( + pk varchar(30) primary key, + f1 integer, + f2 integer, + f3 integer, + FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), + FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), + FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) +); + +CREATE TABLE "custom_foreign_key_label" ( + pk varchar(30) primary key, + foreign_key_with_custom_label text, + FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) +); + +CREATE TABLE tags ( + tag TEXT PRIMARY KEY +); + +CREATE TABLE searchable ( + pk integer primary key, + text1 text, + text2 text, + [name with . and spaces] text +); + +CREATE TABLE searchable_tags ( + searchable_id integer, + tag text, + PRIMARY KEY (searchable_id, tag), + FOREIGN KEY (searchable_id) REFERENCES searchable(pk), + FOREIGN KEY (tag) REFERENCES tags(tag) +); + +INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); +INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); + +INSERT INTO tags VALUES ("canine"); +INSERT INTO tags VALUES ("feline"); + +INSERT INTO searchable_tags (searchable_id, tag) VALUES + (1, "feline"), + (2, "canine") +; + +CREATE VIRTUAL TABLE "searchable_fts" + USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); +INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); + +CREATE TABLE [select] ( + [group] text, + [having] text, + [and] text, + [json] text +); +INSERT INTO [select] VALUES ('group', 'having', 'and', + '{"href": "http://example.com/", "label":"Example"}' +); + +CREATE TABLE infinity ( + value REAL +); +INSERT INTO infinity VALUES + (1e999), + (-1e999), + (1.5) +; + +CREATE TABLE facet_cities ( + id integer primary key, + name text +); +INSERT INTO facet_cities (id, name) VALUES + (1, 'San Francisco'), + (2, 'Los Angeles'), + (3, 'Detroit'), + (4, 'Memnonia') +; + +CREATE TABLE facetable ( + pk integer primary key, + created text, + planet_int integer, + on_earth integer, + state text, + _city_id integer, + _neighborhood text, + tags text, + complex_array text, + distinct_some_null, + n text, + FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) +); +INSERT INTO facetable + (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) +VALUES + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) +; + +CREATE TABLE binary_data ( + data BLOB +); + +-- Many 2 Many demo: roadside attractions! + +CREATE TABLE roadside_attractions ( + pk integer primary key, + name text, + address text, + url text, + latitude real, + longitude real +); +INSERT INTO roadside_attractions VALUES ( + 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", + 37.0167, -122.0024 +); +INSERT INTO roadside_attractions VALUES ( + 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", + 37.3184, -121.9511 +); +INSERT INTO roadside_attractions VALUES ( + 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, + 37.5793, -122.3442 +); +INSERT INTO roadside_attractions VALUES ( + 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", + 37.0414, -122.0725 +); + +CREATE TABLE attraction_characteristic ( + pk integer primary key, + name text +); +INSERT INTO attraction_characteristic VALUES ( + 1, "Museum" +); +INSERT INTO attraction_characteristic VALUES ( + 2, "Paranormal" +); + +CREATE TABLE roadside_attraction_characteristics ( + attraction_id INTEGER REFERENCES roadside_attractions(pk), + characteristic_id INTEGER REFERENCES attraction_characteristic(pk) +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 1, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 2, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 4, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 3, 1 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 4, 1 +); + +INSERT INTO simple_primary_key VALUES (1, 'hello'); +INSERT INTO simple_primary_key VALUES (2, 'world'); +INSERT INTO simple_primary_key VALUES (3, ''); +INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); +INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); + +INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); +INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); + +INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); +INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); + +INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); +INSERT INTO custom_foreign_key_label VALUES (1, 1); + +INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); + +CREATE VIEW simple_view AS + SELECT content, upper(content) AS upper_content FROM simple_primary_key; + +CREATE VIEW searchable_view AS + SELECT * from searchable; + +CREATE VIEW searchable_view_configured_by_metadata AS + SELECT * from searchable; + +""" + + "\n".join( + [ + 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( + i=i + 1 + ) + for i in range(201) + ] + ) + + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + + "\n".join( + [ + 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + a=a, b=b, c=c, content=content + ) + for a, b, c, content in generate_compound_rows(1001) + ] + ) + + "\n".join(["""INSERT INTO sortable VALUES ( + "{pk1}", "{pk2}", "{content}", {sortable}, + {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); + """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) +) + +TABLE_PARAMETERIZED_SQL = [ + ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (null);", []), +] + +EXTRA_DATABASE_SQL = """ +CREATE TABLE searchable ( + pk integer primary key, + text1 text, + text2 text +); + +CREATE VIEW searchable_view AS SELECT * FROM searchable; + +INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); +INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); + +CREATE VIRTUAL TABLE "searchable_fts" + USING FTS3 (text1, text2, content="searchable"); +INSERT INTO "searchable_fts" (rowid, text1, text2) + SELECT rowid, text1, text2 FROM searchable; +""" + + +@documented(label="datasette_fixtures_populate_fixture_database") +def populate_fixture_database(conn): + """Populate a SQLite connection with Datasette's test fixture tables.""" + conn.executescript(TABLES) + for sql, params in TABLE_PARAMETERIZED_SQL: + with conn: + conn.execute(sql, params) + + +def populate_extra_database(conn): + """Populate a SQLite connection with the extra database used in tests.""" + conn.executescript(EXTRA_DATABASE_SQL) + + +def write_fixture_database(db_filename): + """Write Datasette's test fixture tables to a SQLite database file.""" + conn = sqlite3.connect(db_filename) + try: + populate_fixture_database(conn) + finally: + conn.close() + + +def write_extra_database(db_filename): + """Write the extra test database tables to a SQLite database file.""" + conn = sqlite3.connect(db_filename) + try: + populate_extra_database(conn) + finally: + conn.close() diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index eedb2481..dcd502af 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -55,7 +55,17 @@ def publish_subcommand(publish): @hookspec -def render_cell(row, value, column, table, database, datasette, request): +def render_cell( + row, + value, + column, + table, + pks, + database, + datasette, + request, + column_type, +): """Customize rendering of HTML table cell values""" @@ -70,8 +80,13 @@ def register_facet_classes(): @hookspec -def register_permissions(datasette): - """Register permissions: returns a list of datasette.permission.Permission named tuples""" +def register_actions(datasette): + """Register actions: returns a list of datasette.permission.Action objects""" + + +@hookspec +def register_column_types(datasette): + """Return a list of ColumnType subclasses""" @hookspec @@ -110,28 +125,18 @@ def filters_from_request(request, database, table, datasette): ) based on the request""" -@hookspec -def permission_allowed(datasette, actor, action, resource): - """Check if actor is allowed to perform this action - return True, False or None""" - - @hookspec def permission_resources_sql(datasette, actor, action): """Return SQL query fragments for permission checks on resources. - Returns None, a PluginSQL object, or a list of PluginSQL objects. - Each PluginSQL contains SQL that should return rows with columns: + Returns None, a PermissionSQL object, or a list of PermissionSQL objects. + Each PermissionSQL contains SQL that should return rows with columns: parent (str|None), child (str|None), allow (int), reason (str). Used to efficiently check permissions across multiple resources at once. """ -@hookspec -def canned_queries(datasette, database, actor): - """Return a dictionary of canned query definitions or an awaitable function that returns them""" - - @hookspec def register_magic_parameters(datasette): """Return a list of (name, function) magic parameter functions""" @@ -147,6 +152,11 @@ def menu_links(datasette, actor, request): """Links for the navigation menu""" +@hookspec +def jump_items_sql(datasette, actor, request): + """SQL fragments for extra items in the jump menu""" + + @hookspec def row_actions(datasette, actor, request, database, table, row): """Links for the row actions menu""" @@ -164,7 +174,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and canned query actions menu""" + """Links for the query and stored query actions menu""" @hookspec @@ -177,11 +187,6 @@ def homepage_actions(datasette, actor, request): """Links for the homepage actions menu""" -@hookspec -def skip_csrf(datasette, scope): - """Mechanism for skipping CSRF checks for certain requests""" - - @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" @@ -223,5 +228,38 @@ def top_query(datasette, request, database, sql): @hookspec -def top_canned_query(datasette, request, database, query_name): - """HTML to include at the top of the canned query page""" +def top_stored_query(datasette, request, database, query_name): + """HTML to include at the top of the stored query page""" + + +@hookspec +def register_token_handler(datasette): + """Return a TokenHandler instance for token creation and verification""" + + +@hookspec +def write_wrapper(datasette, database, request, transaction): + """Called when a write function is about to execute. + + Return a generator function that accepts a ``conn`` argument and + optionally a ``track_event`` argument. The generator should + ``yield`` exactly once: code before the ``yield`` runs before + the write, code after the ``yield`` runs after the write + completes. The result of the write is sent back through the + ``yield``, so you can capture it with ``result = yield``. + + If your generator accepts ``track_event``, you can call + ``track_event(event)`` to queue an event that will be dispatched + via ``datasette.track_event()`` after the write commits + successfully. Events are discarded if the write raises an + exception. + + If the write raises an exception, it is thrown into the generator + so you can handle it with a try/except around the ``yield``. + + ``request`` may be ``None`` for writes not originating from an + HTTP request. ``transaction`` is ``True`` if the write will + be wrapped in a transaction. + + Return ``None`` to skip wrapping. + """ diff --git a/datasette/inspect.py b/datasette/inspect.py index ede142d0..5e681e03 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -10,7 +10,6 @@ from .utils import ( sqlite3, ) - HASH_BLOCK_SIZE = 1024 * 1024 @@ -70,16 +69,11 @@ def inspect_tables(conn, database_metadata): tables[table]["foreign_keys"] = info # Mark tables 'hidden' if they relate to FTS virtual tables - hidden_tables = [ - r["name"] - for r in conn.execute( - """ + hidden_tables = [r["name"] for r in conn.execute(""" select name from sqlite_master where rootpage = 0 and sql like '%VIRTUAL TABLE%USING FTS%' - """ - ) - ] + """)] if detect_spatialite(conn): # Also hide Spatialite internal tables @@ -94,14 +88,11 @@ def inspect_tables(conn, database_metadata): "views_geometry_columns", "virts_geometry_columns", ] + [ - r["name"] - for r in conn.execute( - """ + r["name"] for r in conn.execute(""" select name from sqlite_master where name like "idx_%" and type = "table" - """ - ) + """) ] for t in tables.keys(): diff --git a/datasette/jump.py b/datasette/jump.py new file mode 100644 index 00000000..d138e827 --- /dev/null +++ b/datasette/jump.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + + +@dataclass +class JumpSQL: + sql: str + params: dict[str, Any] | None = None + database: str | None = None + + @classmethod + def menu_item( + cls, + *, + label: str, + url: str, + description: str = "Menu item", + search_text: str | None = None, + display_name: str | None = None, + item_type: str = "menu", + ) -> "JumpSQL": + if search_text is None: + search_text = " ".join( + text for text in (label, display_name, description) if text is not None + ) + return cls( + sql=""" + SELECT + :type AS type, + :label AS label, + :description AS description, + :url AS url, + :search_text AS search_text, + :display_name AS display_name + """, + params={ + "type": item_type, + "label": label, + "description": description, + "url": url, + "search_text": search_text, + "display_name": display_name, + }, + ) + + +_PARAM_RE = re.compile(r"(? str: + return "/".join( + str(part) for part in (self.parent, self.child) if part is not None + ) + + def __repr__(self) -> str: + return "{}(parent={!r}, child={!r})".format( + self.__class__.__name__, self.parent, self.child + ) + + @property + def private(self) -> bool: + """ + Whether this resource is private (accessible to actor but not anonymous). + + This property is only available on Resource objects returned from + allowed_resources() when include_is_private=True is used. + + Raises: + AttributeError: If accessed without calling include_is_private=True + """ + if self._private is None: + raise AttributeError( + "The 'private' attribute is only available when using " + "allowed_resources(..., include_is_private=True)" + ) + return self._private + + @private.setter + def private(self, value: bool): + self._private = value + + @classmethod + def __init_subclass__(cls): + """ + Validate resource hierarchy doesn't exceed 2 levels. + + Raises: + ValueError: If this resource would create a 3-level hierarchy + """ + super().__init_subclass__() + + if cls.parent_class is None: + return # Top of hierarchy, nothing to validate + + # Check if our parent has a parent - that would create 3 levels + if cls.parent_class.parent_class is not None: + # We have a parent, and that parent has a parent + # This creates a 3-level hierarchy, which is not allowed + raise ValueError( + f"Resource {cls.__name__} creates a 3-level hierarchy: " + f"{cls.parent_class.parent_class.__name__} -> {cls.parent_class.__name__} -> {cls.__name__}. " + f"Maximum 2 levels allowed (parent -> child)." + ) + + @classmethod + @abstractmethod + async def resources_sql(cls, datasette, actor=None) -> str: + """ + Return SQL query that returns all resources of this type. + + Must return two columns: parent, child + """ + pass + + +class AllowedResource(NamedTuple): + """A resource with the reason it was allowed (for debugging).""" + + resource: Resource + reason: str + + +@dataclass(frozen=True, kw_only=True) +class Action: + name: str + description: str | None + abbr: str | None = None + resource_class: type[Resource] | None = None + also_requires: str | None = None # Optional action name that must also be allowed + + @property + def takes_parent(self) -> bool: + """ + Whether this action requires a parent identifier when instantiating its resource. + + Returns False for global-only actions (no resource_class). + Returns True for all actions with a resource_class (all resources require a parent identifier). + """ + return self.resource_class is not None + + @property + def takes_child(self) -> bool: + """ + Whether this action requires a child identifier when instantiating its resource. + + Returns False for global actions (no resource_class). + Returns False for parent-level resources (DatabaseResource - parent_class is None). + Returns True for child-level resources (TableResource, QueryResource - have a parent_class). + """ + if self.resource_class is None: + return False + return self.resource_class.parent_class is not None + + +_reason_id = 1 +@dataclass +class PermissionSQL: + """ + A plugin contributes SQL that yields: + parent TEXT NULL, + child TEXT NULL, + allow INTEGER, -- 1 allow, 0 deny + reason TEXT + + For restriction-only plugins, sql can be None and only restriction_sql is provided. + """ + + sql: str | None = ( + None # SQL that SELECTs the 4 columns above (can be None for restriction-only) + ) + params: dict[str, Any] | None = ( + None # bound params for the SQL (values only; no ':' prefix) + ) + source: str | None = None # System will set this to the plugin name + restriction_sql: str | None = ( + None # Optional SQL that returns (parent, child) for restriction filtering + ) + + @classmethod + def allow(cls, reason: str, _allow: bool = True) -> "PermissionSQL": + global _reason_id + i = _reason_id + _reason_id += 1 + return cls( + sql=f"SELECT NULL AS parent, NULL AS child, {1 if _allow else 0} AS allow, :reason_{i} AS reason", + params={f"reason_{i}": reason}, + ) + + @classmethod + def deny(cls, reason: str) -> "PermissionSQL": + return cls.allow(reason=reason, _allow=False) + + +# This is obsolete, replaced by Action and ResourceType @dataclass class Permission: name: str - abbr: Optional[str] - description: Optional[str] + abbr: str | None + description: str | None takes_database: bool takes_resource: bool default: bool diff --git a/datasette/plugins.py b/datasette/plugins.py index 3769a209..5a31cdad 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -23,9 +23,14 @@ DEFAULT_PLUGINS = ( "datasette.sql_functions", "datasette.actor_auth_cookie", "datasette.default_permissions", + "datasette.default_permissions.tokens", + "datasette.default_actions", + "datasette.default_column_types", "datasette.default_magic_parameters", "datasette.blob_renderer", - "datasette.default_menu_links", + "datasette.default_debug_menu", + "datasette.default_jump_items", + "datasette.default_database_actions", "datasette.handle_exception", "datasette.forbidden", "datasette.events", @@ -49,7 +54,7 @@ def after(outcome, hook_name, hook_impls, kwargs): results = outcome.get_result() if not isinstance(results, list): results = [results] - print(f"Results:", file=sys.stderr) + print("Results:", file=sys.stderr) pprint(results, width=40, indent=4, stream=sys.stderr) @@ -93,21 +98,24 @@ def get_plugins(): for plugin in pm.get_plugins(): static_path = None templates_path = None - if plugin.__name__ not in DEFAULT_PLUGINS: + plugin_name = ( + plugin.__name__ + if hasattr(plugin, "__name__") + else plugin.__class__.__name__ + ) + if plugin_name not in DEFAULT_PLUGINS: try: - if (importlib_resources.files(plugin.__name__) / "static").is_dir(): - static_path = str( - importlib_resources.files(plugin.__name__) / "static" - ) - if (importlib_resources.files(plugin.__name__) / "templates").is_dir(): + if (importlib_resources.files(plugin_name) / "static").is_dir(): + static_path = str(importlib_resources.files(plugin_name) / "static") + if (importlib_resources.files(plugin_name) / "templates").is_dir(): templates_path = str( - importlib_resources.files(plugin.__name__) / "templates" + importlib_resources.files(plugin_name) / "templates" ) except (TypeError, ModuleNotFoundError): # Caused by --plugins_dir= plugins pass plugin_info = { - "name": plugin.__name__, + "name": plugin_name, "static_path": static_path, "templates_path": templates_path, "hooks": [h.name for h in pm.get_hookcallers(plugin)], diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 11721039..63d22fe8 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -23,7 +23,9 @@ def publish_subcommand(publish): help="Application name to use when building", ) @click.option( - "--service", default="", help="Cloud Run service to deploy (or over-write)" + "--service", + default="", + help="Cloud Run service to deploy (or over-write)", ) @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") @click.option( @@ -301,6 +303,7 @@ def get_existing_services(): "url": service["status"]["address"]["url"], } for service in services + if "url" in service["status"] ] diff --git a/datasette/renderer.py b/datasette/renderer.py index 483c81e9..acf23e59 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -20,7 +20,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): if column in json_cols: try: value = json.loads(value) - except (TypeError, ValueError) as e: + except (TypeError, ValueError): pass new_row.append(value) new_rows.append(new_row) diff --git a/datasette/resources.py b/datasette/resources.py new file mode 100644 index 00000000..ee2e6d98 --- /dev/null +++ b/datasette/resources.py @@ -0,0 +1,58 @@ +"""Core resource types for Datasette's permission system.""" + +from datasette.permissions import Resource + + +class DatabaseResource(Resource): + """A database in Datasette.""" + + name = "database" + parent_class = None # Top of the resource hierarchy + + def __init__(self, database: str): + super().__init__(parent=database, child=None) + + @classmethod + async def resources_sql(cls, datasette, actor=None) -> str: + return """ + SELECT database_name AS parent, NULL AS child + FROM catalog_databases + """ + + +class TableResource(Resource): + """A table in a database.""" + + name = "table" + parent_class = DatabaseResource + + def __init__(self, database: str, table: str): + super().__init__(parent=database, child=table) + + @classmethod + async def resources_sql(cls, datasette, actor=None) -> str: + return """ + SELECT database_name AS parent, table_name AS child + FROM catalog_tables + UNION ALL + SELECT database_name AS parent, view_name AS child + FROM catalog_views + """ + + +class QueryResource(Resource): + """A stored query in a database.""" + + name = "query" + parent_class = DatabaseResource + + def __init__(self, database: str, query: str): + super().__init__(parent=database, child=query) + + @classmethod + async def resources_sql(cls, datasette, actor=None) -> str: + return """ + SELECT q.database_name AS parent, q.name AS child + FROM queries q + JOIN catalog_databases cd ON cd.database_name = q.database_name + """ diff --git a/datasette/static/app.css b/datasette/static/app.css index a3117152..815f6db8 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -63,6 +63,14 @@ em { } /* end reset */ +/* Modal CSS variables (shared by web components via Shadow DOM) */ +:root { + --modal-backdrop-bg: rgba(0, 0, 0, 0.5); + --modal-backdrop-blur: blur(4px); + --modal-border-radius: 0.75rem; + --modal-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); + --modal-animation-duration: 0.2s; +} body { margin: 0; @@ -354,6 +362,32 @@ form.nav-menu-logout { .nav-menu-inner a { display: block; } +.nav-menu-inner button.button-as-link { + display: block; + width: 100%; + text-align: left; + font: inherit; +} +.nav-menu-inner .keyboard-shortcut { + float: right; + box-sizing: border-box; + min-width: 1.4em; + margin-left: 0.75rem; + padding: 0 0.35em; + border: 1px solid rgba(255,255,244,0.6); + border-radius: 3px; + background: rgba(255,255,244,0.12); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.85em; + line-height: 1.35; + text-align: center; + text-decoration: none; +} +@media (max-width: 640px) { + .nav-menu-inner .keyboard-shortcut { + display: none; + } +} /* Table/database actions menu */ .page-action-menu { @@ -647,10 +681,14 @@ button.core[type=button] { border-radius: 3px; -webkit-appearance: none; padding: 9px 4px; - font-size: 1em; + font-size: 16px; font-family: Helvetica, sans-serif; } +#_search { + font-size: 16px; +} + @@ -730,6 +768,474 @@ p.zero-results { .select-wrapper.small-screen-only { display: none; } + +@keyframes datasette-modal-slide-in { + from { + opacity: 0; + transform: translateY(-20px) scale(0.95); + } + to { + opacity: 1; + transform: translateY(0) scale(1); + } +} + +@keyframes datasette-modal-fade-in { + from { opacity: 0; } + to { opacity: 1; } +} + +dialog.mobile-column-actions-dialog { + --ink: #0f0f0f; + --paper: #f5f3ef; + --muted: #6b6b6b; + --rule: #e2dfd8; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(420px, calc(100vw - 32px)); + max-width: 95vw; + max-height: min(640px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.mobile-column-actions-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.mobile-column-actions-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.mobile-column-actions-dialog .modal-header { + padding: 20px 24px 16px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.mobile-column-actions-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.mobile-column-actions-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; +} + +.mobile-column-actions-dialog .list-wrap { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; + overflow-x: hidden; + position: relative; + overscroll-behavior: contain; + -webkit-overflow-scrolling: touch; +} + +.mobile-column-actions-dialog .list-wrap::before, +.mobile-column-actions-dialog .list-wrap::after { + content: ""; + position: sticky; + display: block; + left: 0; + right: 0; + height: 20px; + pointer-events: none; + z-index: 5; +} + +.mobile-column-actions-dialog .list-wrap::before { + top: 0; + background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent); +} + +.mobile-column-actions-dialog .list-wrap::after { + bottom: 0; + background: linear-gradient(to top, rgba(255,255,255,0.9), transparent); + margin-top: -20px; +} + +.mobile-column-top-actions { + padding: 10px 24px 0; +} + +.mobile-column-top-action { + display: inline-block; + text-decoration: none; +} + +.mobile-column-section { + border-bottom: 1px solid var(--rule); +} + +.mobile-column-actions-dialog .col-header { + width: 100%; + padding: 12px 24px; + font: inherit; + font-weight: 600; + border: 0; + background: none; + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; + text-align: left; +} + +.mobile-column-header-text { + display: flex; + flex-direction: column; + gap: 0.15rem; +} + +.mobile-column-name { + color: var(--ink); +} + +.mobile-column-meta { + color: var(--muted); + font-size: 0.78em; + font-family: ui-monospace, monospace; + font-weight: normal; +} + +.mobile-column-chevron { + color: var(--muted); + transition: transform 0.2s ease-out; +} + +.mobile-column-actions-dialog .col-header[aria-expanded="true"] .mobile-column-chevron { + transform: rotate(180deg); +} + +.mobile-column-actions-dialog .col-actions[hidden] { + display: none; +} + +.mobile-column-actions-dialog .col-actions ul, +.mobile-column-actions-dialog .col-actions li { + margin: 0; + padding: 0; + list-style-type: none; +} + +.mobile-column-actions-dialog .col-actions a, +.mobile-column-actions-dialog .col-actions button { + display: block; + width: 100%; + padding: 10px 24px 10px 40px; + color: var(--ink); + text-align: left; + font: inherit; + text-decoration: none; + background: none; + border: 0; + border-top: 1px solid #f5f5f5; + cursor: pointer; +} + +.mobile-column-actions-dialog .col-actions a:hover, +.mobile-column-actions-dialog .col-actions button:hover { + background: var(--paper); +} + +.mobile-column-actions-dialog .col-actions a:active, +.mobile-column-actions-dialog .col-actions button:active { + background: #eee; +} + +.mobile-column-description, +.mobile-column-no-actions { + margin: 0; + padding: 0 24px 12px 24px; + color: var(--muted); + font-size: 0.85em; +} + +.mobile-column-actions-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.mobile-column-actions-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.mobile-column-actions-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.mobile-column-actions-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.mobile-column-actions-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +dialog.set-column-type-dialog { + --ink: #0f0f0f; + --paper: #f5f3ef; + --muted: #6b6b6b; + --rule: #e2dfd8; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(520px, calc(100vw - 32px)); + max-width: 95vw; + max-height: min(720px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.set-column-type-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.set-column-type-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.set-column-type-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.set-column-type-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.set-column-type-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; +} + +.set-column-type-status, +.set-column-type-empty, +.set-column-type-error { + margin: 0; + padding: 12px 24px 0; +} + +.set-column-type-status, +.set-column-type-empty { + color: var(--muted); + font-size: 0.9rem; +} + +.set-column-type-error { + color: #b91c1c; + font-size: 0.9rem; +} + +.set-column-type-options { + padding: 16px 24px 24px; + overflow-y: auto; + display: grid; + gap: 12px; +} + +.set-column-type-option { + display: grid; + grid-template-columns: auto 1fr; + gap: 12px; + align-items: start; + padding: 14px 16px; + border: 1px solid var(--rule); + border-radius: 8px; + background: #fcfbf9; + cursor: pointer; +} + +.set-column-type-option:focus-within { + border-color: var(--accent); + box-shadow: 0 0 0 3px rgba(26, 86, 219, 0.12); +} + +.set-column-type-option input { + margin-top: 3px; +} + +.set-column-type-option-content { + display: grid; + gap: 4px; +} + +.set-column-type-option-name { + font-family: ui-monospace, monospace; + font-size: 0.95rem; + color: var(--ink); +} + +.set-column-type-option-description { + color: var(--muted); + font-size: 0.9rem; +} + +.set-column-type-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.set-column-type-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.set-column-type-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.set-column-type-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.set-column-type-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.set-column-type-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.set-column-type-dialog .btn-primary:hover { + background: #1949b8; +} + +.set-column-type-dialog .btn:disabled { + opacity: 0.65; + cursor: wait; +} + +@media (max-width: 640px) { + dialog.mobile-column-actions-dialog { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; + } + + .mobile-column-actions-dialog .modal-header { + padding: 16px 18px 14px; + } + + .mobile-column-top-actions { + padding-left: 18px; + padding-right: 18px; + } + + .mobile-column-actions-dialog .col-header { + padding-left: 18px; + padding-right: 18px; + } + + .mobile-column-actions-dialog .col-actions a, + .mobile-column-actions-dialog .col-actions button { + padding-left: 34px; + padding-right: 18px; + } + + .mobile-column-description, + .mobile-column-no-actions { + padding-left: 18px; + padding-right: 18px; + } + + dialog.set-column-type-dialog { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; + } + + .set-column-type-dialog .modal-header, + .set-column-type-status, + .set-column-type-empty, + .set-column-type-error, + .set-column-type-options { + padding-left: 18px; + padding-right: 18px; + } +} + @media only screen and (max-width: 576px) { .small-screen-only { @@ -791,6 +1297,43 @@ p.zero-results { .filters input.filter-value { width: 140px; } + button.choose-columns-mobile, + button.column-actions-mobile { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0.5rem 1rem; + margin-bottom: 1em; + font-size: 0.9rem; + line-height: 1.2; + font-family: inherit; + background: white; + border: 1px solid #ccc; + border-radius: 5px; + cursor: pointer; + vertical-align: top; + box-sizing: border-box; + min-height: 2.5rem; + } + + button.column-actions-mobile { + gap: 0.55rem; + } + + button.column-actions-mobile svg { + display: block; + width: 16px; + height: 16px; + flex-shrink: 0; + } + + button.column-actions-mobile span { + line-height: 1.2; + } + + button.choose-columns-mobile { + margin-right: 0.5rem; + } } svg.dropdown-menu-icon { @@ -866,11 +1409,15 @@ svg.dropdown-menu-icon { border-bottom: 5px solid #666; } -.canned-query-edit-sql { +.stored-query-edit-sql { padding-left: 0.5em; position: relative; top: 1px; } +.save-query { + display: inline-block; + margin-left: 0.45em; +} .blob-download { display: block; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js new file mode 100644 index 00000000..133e7cb0 --- /dev/null +++ b/datasette/static/column-chooser.js @@ -0,0 +1,699 @@ +class ColumnChooser extends HTMLElement { + constructor() { + super(); + this.attachShadow({ mode: "open" }); + + // State + this._items = []; + this._checked = new Set(); + this._savedItems = null; + this._savedChecked = null; + this._onApply = null; + + // Drag state + this._ghost = null; + this._dragSrcIdx = null; + this._dropTargetIdx = null; + this._dropPosition = null; + this._ghostOffX = 0; + this._ghostOffY = 0; + this._autoScrollRAF = null; + this._lastPointerY = 0; + this._lastPointerX = 0; + this._SCROLL_ZONE = 72; + this._SCROLL_SPEED = 0.4; + + // Bound handlers + this._onMove = this._onMove.bind(this); + this._onUp = this._onUp.bind(this); + + this.shadowRoot.innerHTML = ` + + + +
+ Choose columns + +
+
+ + +
+
+
+
+
    +
    +
    + + + +
    +     + `; + + // DOM refs + this._dialog = this.shadowRoot.querySelector("dialog"); + this._listWrap = this.shadowRoot.getElementById("listWrap"); + this._dragList = this.shadowRoot.getElementById("dragList"); + this._pulseTop = this.shadowRoot.getElementById("pulseTop"); + this._pulseBot = this.shadowRoot.getElementById("pulseBot"); + this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); + this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); + this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); + this._applyBtn = this.shadowRoot.getElementById("applyBtn"); + this._countEl = this.shadowRoot.getElementById("selectedCount"); + this._footerEl = this.shadowRoot.getElementById("footerInfo"); + + // Event listeners + this._selectAllBtn.addEventListener("click", () => this._selectAll()); + this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); + this._cancelBtn.addEventListener("click", () => this._close()); + this._applyBtn.addEventListener("click", () => this._apply()); + this._dialog.addEventListener("click", (e) => { + if (e.target === this._dialog) this._close(); + }); + this._dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this._close(); + }); + } + + /** + * Open the column chooser dialog. + * @param {Object} opts + * @param {string[]} opts.columns - All available column names, in display order. + * @param {string[]} opts.selected - Column names that should be pre-checked. + * @param {function(string[]): void} opts.onApply - Called with the selected columns in order when Apply is clicked. + */ + open({ columns, selected = [], onApply }) { + this._items = [...columns]; + this._checked = new Set(selected); + this._onApply = onApply || null; + + // Save state for cancel/restore + this._savedItems = [...this._items]; + this._savedChecked = new Set(this._checked); + + this._render(); + this._dialog.showModal(); + } + + // ── Internal methods ── + + _close() { + this._items = this._savedItems ? [...this._savedItems] : this._items; + this._checked = this._savedChecked + ? new Set(this._savedChecked) + : this._checked; + this._dialog.close(); + } + + _selectAll() { + this._items.forEach((col) => this._checked.add(col)); + this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { + cb.checked = true; + }); + this._updateCounts(); + } + + _deselectAll() { + this._checked.clear(); + this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { + cb.checked = false; + }); + this._updateCounts(); + } + + _apply() { + const selected = this._items.filter((col) => this._checked.has(col)); + this._dialog.close(); + if (this._onApply) { + this._onApply(selected); + } + } + + _render() { + this._dragList.innerHTML = ""; + this._items.forEach((col, i) => { + const li = document.createElement("li"); + li.className = "drag-item"; + li.dataset.idx = i; + li.innerHTML = ` + + + + + + + + + + + +
    + `; + + li.querySelector("input").addEventListener("change", (e) => { + e.target.checked ? this._checked.add(col) : this._checked.delete(col); + this._updateCounts(); + }); + + li.querySelector(".drag-handle").addEventListener("pointerdown", (e) => + this._startDrag(e, i), + ); + this._dragList.appendChild(li); + }); + + this._updateCounts(); + } + + _updateCounts() { + const n = this._checked.size; + this._countEl.textContent = `${n} of ${this._items.length} selected`; + this._footerEl.textContent = `${this._items.length} columns`; + } + + // ── Drag engine ── + + _startDrag(e, idx) { + e.preventDefault(); + this._dragSrcIdx = idx; + + const srcEl = this._dragList.children[idx]; + const rect = srcEl.getBoundingClientRect(); + + this._ghostOffX = e.clientX - rect.left; + this._ghostOffY = e.clientY - rect.top; + + // Build ghost inside shadow DOM + this._ghost = document.createElement("div"); + this._ghost.className = "drag-ghost"; + this._ghost.style.width = rect.width + "px"; + this._ghost.style.height = rect.height + "px"; + this._ghost.innerHTML = srcEl.innerHTML; + this._ghost.querySelector(".drop-indicator")?.remove(); + const h = this._ghost.querySelector(".drag-handle"); + if (h) h.style.color = "var(--accent)"; + this.shadowRoot.appendChild(this._ghost); + + srcEl.classList.add("is-dragging"); + this._positionGhost(e.clientX, e.clientY); + + document.addEventListener("pointermove", this._onMove); + document.addEventListener("pointerup", this._onUp); + document.addEventListener("pointercancel", this._onUp); + } + + _positionGhost(cx, cy) { + this._ghost.style.left = cx - this._ghostOffX + "px"; + this._ghost.style.top = cy - this._ghostOffY + "px"; + } + + _onMove(e) { + this._lastPointerX = e.clientX; + this._lastPointerY = e.clientY; + this._positionGhost(e.clientX, e.clientY); + this._updateDropTarget(e.clientY); + this._updateAutoScroll(e.clientY); + } + + _onUp() { + document.removeEventListener("pointermove", this._onMove); + document.removeEventListener("pointerup", this._onUp); + document.removeEventListener("pointercancel", this._onUp); + + this._stopAutoScroll(); + + const noMove = + this._dropTargetIdx === null || this._dropTargetIdx === this._dragSrcIdx; + this._clearDropIndicators(); + + let dest = null; + if (!noMove) { + const moved = this._items.splice(this._dragSrcIdx, 1)[0]; + dest = this._dropTargetIdx; + if (this._dropPosition === "after") dest++; + if (dest > this._dragSrcIdx) dest--; + this._items.splice(dest, 0, moved); + } + + this._dragSrcIdx = null; + this._dropTargetIdx = null; + this._dropPosition = null; + + const g = this._ghost; + this._ghost = null; + + if (noMove) { + if (g) g.remove(); + this._render(); + return; + } + + this._render(); + + if (g && dest !== null) { + const landedEl = this._dragList.children[dest]; + if (landedEl) { + landedEl.style.opacity = "0"; + const r = landedEl.getBoundingClientRect(); + g.getBoundingClientRect(); + g.style.transition = + "left 0.15s cubic-bezier(0.22, 1, 0.36, 1), top 0.15s cubic-bezier(0.22, 1, 0.36, 1), box-shadow 0.15s, opacity 0.1s 0.1s"; + g.style.left = r.left + "px"; + g.style.top = r.top + "px"; + g.style.boxShadow = "0 1px 4px rgba(0,0,0,0.08)"; + g.style.opacity = "0"; + setTimeout(() => { + g.remove(); + if (landedEl) landedEl.style.opacity = ""; + }, 160); + } else { + g.remove(); + } + } else if (g) { + g.remove(); + } + } + + _updateDropTarget(clientY) { + this._clearDropIndicators(); + const listItems = [ + ...this._dragList.querySelectorAll(".drag-item:not(.is-dragging)"), + ]; + if (!listItems.length) return; + + let best = null, + bestDist = Infinity; + listItems.forEach((li) => { + const r = li.getBoundingClientRect(); + const mid = r.top + r.height / 2; + const dist = Math.abs(clientY - mid); + if (dist < bestDist) { + bestDist = dist; + best = li; + } + }); + + if (!best) return; + const r = best.getBoundingClientRect(); + const mid = r.top + r.height / 2; + const above = clientY < mid; + const indic = best.querySelector(".drop-indicator"); + + this._dropTargetIdx = parseInt(best.dataset.idx); + this._dropPosition = above ? "before" : "after"; + + if (indic) { + indic.className = "drop-indicator " + (above ? "top" : "bottom"); + } + } + + _clearDropIndicators() { + this._dragList.querySelectorAll(".drop-indicator").forEach((el) => { + el.className = "drop-indicator"; + }); + } + + _updateAutoScroll(clientY) { + const rect = this._listWrap.getBoundingClientRect(); + const relY = clientY - rect.top; + const distTop = relY; + const distBot = rect.height - relY; + + const inTop = distTop < this._SCROLL_ZONE && distTop >= 0; + const inBot = distBot < this._SCROLL_ZONE && distBot >= 0; + + this._pulseTop.classList.toggle("active", inTop); + this._pulseBot.classList.toggle("active", inBot); + + if ((inTop || inBot) && !this._autoScrollRAF) { + let lastTime = null; + const loop = (ts) => { + if (!this._ghost) { + this._stopAutoScroll(); + return; + } + if (lastTime !== null) { + const dt = ts - lastTime; + const rect2 = this._listWrap.getBoundingClientRect(); + const relY2 = this._lastPointerY - rect2.top; + const dTop = relY2; + const dBot = rect2.height - relY2; + + if (dTop < this._SCROLL_ZONE && dTop >= 0) { + const factor = 1 - dTop / this._SCROLL_ZONE; + this._listWrap.scrollTop -= this._SCROLL_SPEED * dt * factor * 2.5; + } else if (dBot < this._SCROLL_ZONE && dBot >= 0) { + const factor = 1 - dBot / this._SCROLL_ZONE; + this._listWrap.scrollTop += this._SCROLL_SPEED * dt * factor * 2.5; + } else { + this._stopAutoScroll(); + return; + } + this._updateDropTarget(this._lastPointerY); + } + lastTime = ts; + this._autoScrollRAF = requestAnimationFrame(loop); + }; + this._autoScrollRAF = requestAnimationFrame(loop); + } + + if (!inTop && !inBot) this._stopAutoScroll(); + } + + _stopAutoScroll() { + if (this._autoScrollRAF) { + cancelAnimationFrame(this._autoScrollRAF); + this._autoScrollRAF = null; + } + this._pulseTop.classList.remove("active"); + this._pulseBot.classList.remove("active"); + } +} + +customElements.define("column-chooser", ColumnChooser); diff --git a/datasette/static/datasette-manager.js b/datasette/static/datasette-manager.js index 10716cc5..e75f7aae 100644 --- a/datasette/static/datasette-manager.js +++ b/datasette/static/datasette-manager.js @@ -82,6 +82,19 @@ const datasetteManager = { return columnActions; }, + makeJumpSections: (context) => { + let jumpSections = []; + + datasetteManager.plugins.forEach((plugin) => { + if (plugin.makeJumpSections) { + const sections = plugin.makeJumpSections(context) || []; + jumpSections.push(...sections); + } + }); + + return jumpSections; + }, + /** * In MVP, each plugin can only have 1 instance. * In future, panels could be repeated. We omit that for now since so many plugins depend on @@ -93,12 +106,12 @@ const datasetteManager = { */ renderAboveTablePanel: () => { const aboveTablePanel = document.querySelector( - DOM_SELECTORS.aboveTablePanel + DOM_SELECTORS.aboveTablePanel, ); if (!aboveTablePanel) { console.warn( - "This page does not have a table, the renderAboveTablePanel cannot be used." + "This page does not have a table, the renderAboveTablePanel cannot be used.", ); return; } @@ -192,7 +205,6 @@ const initializeDatasette = () => { // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window. window.__DATASETTE__ = datasetteManager; - console.debug("Datasette Manager Created!"); const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, { detail: datasetteManager, diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js index d83b8186..0e6e2c29 100644 --- a/datasette/static/json-format-highlight-1.0.1.js +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -7,8 +7,8 @@ MIT Licensed typeof exports === "object" && typeof module !== "undefined" ? (module.exports = factory()) : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); + ? define(factory) + : (global.jsonFormatHighlight = factory()); })(this, function () { "use strict"; @@ -42,13 +42,13 @@ MIT Licensed color = /true/.test(match) ? colors.trueColor : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; } return '' + match + ""; - } + }, ); } diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js new file mode 100644 index 00000000..a386b1fc --- /dev/null +++ b/datasette/static/mobile-column-actions.js @@ -0,0 +1,318 @@ +var MOBILE_COLUMN_BREAKPOINT = 576; +var MOBILE_COLUMN_DIALOG_ID = "mobile-column-actions-dialog"; +var MOBILE_COLUMN_DIALOG_TITLE_ID = "mobile-column-actions-title"; + +function mobileColumnHeaders(manager) { + return Array.from( + document.querySelectorAll(manager.selectors.tableHeaders), + ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1"); +} + +function mobileColumnMetaText(th) { + var parts = []; + if (th.dataset.columnType) { + parts.push(th.dataset.columnType); + } + if (th.dataset.isPk === "1") { + parts.push("pk"); + } + if (th.dataset.columnNotNull === "1") { + parts.push("not null"); + } + return parts.join(", "); +} + +function createMobileColumnActionNode(itemConfig, closeDialog) { + var actionNode; + if (itemConfig.href) { + actionNode = document.createElement("a"); + actionNode.href = itemConfig.href; + } else { + actionNode = document.createElement("button"); + actionNode.type = "button"; + } + actionNode.textContent = itemConfig.label; + + if (itemConfig.onClick) { + actionNode.addEventListener("click", function (ev) { + try { + itemConfig.onClick.call(actionNode, ev); + } finally { + closeDialog({ restoreFocus: false }); + } + }); + } + + return actionNode; +} + +function initMobileColumnActions(manager) { + var triggerButton = document.querySelector(".column-actions-mobile"); + if (!triggerButton) { + return; + } + + if ( + !window.URLSearchParams || + !window.HTMLDialogElement || + !manager.columnActions + ) { + triggerButton.style.display = "none"; + return; + } + + if (!mobileColumnHeaders(manager).length) { + triggerButton.style.display = "none"; + return; + } + + var dialog = document.createElement("dialog"); + dialog.className = "mobile-column-actions-dialog"; + dialog.id = MOBILE_COLUMN_DIALOG_ID; + dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); + dialog.innerHTML = ` +
    + Column actions + +
    +
    +
    + Tap a column to reveal actions. + +
    + `; + document.body.appendChild(dialog); + + triggerButton.setAttribute("aria-haspopup", "dialog"); + triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); + triggerButton.setAttribute("aria-expanded", "false"); + + var countEl = dialog.querySelector(".modal-meta"); + var listWrap = dialog.querySelector(".mobile-column-list"); + var doneButton = dialog.querySelector(".mobile-column-actions-done"); + var expandedSectionId = null; + var shouldRestoreFocus = true; + + function updateExpandedSection() { + Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { + var controlsId = button.getAttribute("aria-controls"); + var actionList = dialog.querySelector("#" + controlsId); + var isExpanded = controlsId === expandedSectionId; + button.setAttribute("aria-expanded", isExpanded ? "true" : "false"); + actionList.hidden = !isExpanded; + actionList.classList.toggle("expanded", isExpanded); + }); + } + + function scrollExpandedSectionIntoView(section) { + var sectionTop = section.offsetTop; + var sectionBottom = sectionTop + section.offsetHeight; + var visibleTop = listWrap.scrollTop; + var visibleBottom = visibleTop + listWrap.clientHeight; + var sectionHeight = section.offsetHeight; + + if (sectionTop < visibleTop) { + listWrap.scrollTop = sectionTop; + return; + } + + if (sectionBottom <= visibleBottom) { + return; + } + + if (sectionHeight <= listWrap.clientHeight) { + listWrap.scrollTop = sectionBottom - listWrap.clientHeight; + } else { + listWrap.scrollTop = sectionTop; + } + } + + function closeDialog(options) { + options = options || {}; + shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + triggerButton.setAttribute("aria-expanded", "false"); + if (shouldRestoreFocus) { + triggerButton.focus(); + } + } + } + + function renderDialog() { + var headers = mobileColumnHeaders(manager); + if (!headers.length) { + closeDialog({ restoreFocus: false }); + triggerButton.style.display = "none"; + return false; + } + + if ( + !headers.some( + (_th, index) => `mobile-column-actions-${index}` === expandedSectionId, + ) + ) { + expandedSectionId = null; + } + + countEl.textContent = `${headers.length} column${ + headers.length === 1 ? "" : "s" + }`; + listWrap.innerHTML = ""; + + if (manager.columnActions.shouldShowShowAllColumns()) { + var topActions = document.createElement("div"); + topActions.className = "mobile-column-top-actions"; + + var showAllColumns = document.createElement("a"); + showAllColumns.className = "btn btn-ghost mobile-column-top-action"; + showAllColumns.href = manager.columnActions.showAllColumnsUrl(); + showAllColumns.textContent = "Show all columns"; + + topActions.appendChild(showAllColumns); + listWrap.appendChild(topActions); + } + + headers.forEach((th, index) => { + var sectionId = `mobile-column-actions-${index}`; + var actionState = manager.columnActions.buildColumnActionState(th, { + includeChooseColumns: false, + includeShowAllColumns: false, + }); + var section = document.createElement("section"); + section.className = "mobile-column-section"; + + var headerButton = document.createElement("button"); + headerButton.type = "button"; + headerButton.className = "col-header"; + headerButton.setAttribute("aria-controls", sectionId); + headerButton.setAttribute("aria-expanded", "false"); + + var headerText = document.createElement("span"); + headerText.className = "mobile-column-header-text"; + + var name = document.createElement("span"); + name.className = "mobile-column-name"; + name.textContent = th.dataset.column; + headerText.appendChild(name); + + var metaText = mobileColumnMetaText(th); + if (metaText) { + var meta = document.createElement("span"); + meta.className = "mobile-column-meta"; + meta.textContent = metaText; + headerText.appendChild(meta); + } + + var chevron = document.createElement("span"); + chevron.className = "mobile-column-chevron"; + chevron.setAttribute("aria-hidden", "true"); + chevron.textContent = "▾"; + + headerButton.appendChild(headerText); + headerButton.appendChild(chevron); + headerButton.addEventListener("click", function () { + expandedSectionId = expandedSectionId === sectionId ? null : sectionId; + updateExpandedSection(); + if (expandedSectionId === sectionId) { + scrollExpandedSectionIntoView(section); + } + }); + + var actionContainer = document.createElement("div"); + actionContainer.id = sectionId; + actionContainer.className = "col-actions"; + actionContainer.hidden = true; + + if (actionState.columnDescription) { + var description = document.createElement("p"); + description.className = "mobile-column-description"; + description.textContent = actionState.columnDescription; + actionContainer.appendChild(description); + } + + if (actionState.actionItems.length) { + var actionList = document.createElement("ul"); + actionState.actionItems.forEach((itemConfig) => { + var actionItem = document.createElement("li"); + actionItem.appendChild( + createMobileColumnActionNode(itemConfig, closeDialog), + ); + actionList.appendChild(actionItem); + }); + actionContainer.appendChild(actionList); + } else { + var noActions = document.createElement("p"); + noActions.className = "mobile-column-no-actions"; + noActions.textContent = "No actions available"; + actionContainer.appendChild(noActions); + } + + section.appendChild(headerButton); + section.appendChild(actionContainer); + listWrap.appendChild(section); + }); + + updateExpandedSection(); + return true; + } + + function openDialog() { + if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT) { + return; + } + if (!renderDialog()) { + return; + } + if (!dialog.open) { + dialog.showModal(); + } + triggerButton.setAttribute("aria-expanded", "true"); + var focusTarget = + dialog.querySelector(".mobile-column-top-action") || + dialog.querySelector(".col-header") || + doneButton; + focusTarget.focus(); + } + + triggerButton.addEventListener("click", function () { + if (dialog.open) { + closeDialog(); + } else { + openDialog(); + } + }); + + doneButton.addEventListener("click", function () { + closeDialog(); + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeDialog(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeDialog(); + }); + + dialog.addEventListener("close", function () { + triggerButton.setAttribute("aria-expanded", "false"); + if (shouldRestoreFocus) { + triggerButton.focus(); + } + }); + + window.addEventListener("resize", function () { + if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT && dialog.open) { + closeDialog({ restoreFocus: false }); + } + }); +} + +document.addEventListener("datasette_init", function (evt) { + initMobileColumnActions(evt.detail); +}); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js new file mode 100644 index 00000000..ec2d23d8 --- /dev/null +++ b/datasette/static/navigation-search.js @@ -0,0 +1,910 @@ +let navigationSearchInstanceCounter = 0; + +class NavigationSearch extends HTMLElement { + constructor() { + super(); + this.instanceId = ++navigationSearchInstanceCounter; + this.inputId = `navigation-search-input-${this.instanceId}`; + this.instructionsId = `navigation-search-instructions-${this.instanceId}`; + this.listboxId = `navigation-search-results-${this.instanceId}`; + this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; + this.statusId = `navigation-search-status-${this.instanceId}`; + this.titleId = `navigation-search-title-${this.instanceId}`; + this.attachShadow({ mode: "open" }); + this.selectedIndex = -1; + this.matches = []; + this.renderedMatches = []; + this.debounceTimer = null; + this.restoreFocusTarget = null; + this.shouldRestoreFocus = true; + + this.render(); + this.setupEventListeners(); + } + + render() { + this.shadowRoot.innerHTML = ` + + + +
    + + + +
    + + +
    +
    +
    + ↑ ↓ Navigate + Enter Select + Esc Close +
    +
    +     + `; + } + + setupEventListeners() { + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); + const closeButton = this.shadowRoot.querySelector(".close-search"); + const resultsContainer = + this.shadowRoot.querySelector(".results-container"); + + // Global keyboard listener for "/" + document.addEventListener("keydown", (e) => { + if (e.key === "/" && !this.isInputFocused() && !dialog.open) { + e.preventDefault(); + this.openMenu(); + } + }); + + document.addEventListener("click", (e) => { + const trigger = e.target.closest("[data-navigation-search-open]"); + if (trigger) { + e.preventDefault(); + const details = trigger.closest("details"); + const restoreTarget = details?.querySelector("summary") || trigger; + details?.removeAttribute("open"); + this.openMenu(restoreTarget); + } + }); + + // Input event + input.addEventListener("input", (e) => { + this.handleSearch(e.target.value); + }); + + // Keyboard navigation + input.addEventListener("keydown", (e) => { + if (e.key === "ArrowDown") { + e.preventDefault(); + this.moveSelection(1); + } else if (e.key === "ArrowUp") { + e.preventDefault(); + this.moveSelection(-1); + } else if (e.key === "Enter") { + e.preventDefault(); + this.selectCurrentItem(); + } else if (e.key === "Escape") { + this.closeMenu(); + } + }); + + closeButton.addEventListener("click", () => { + this.closeMenu(); + }); + + // Click on result item + resultsContainer.addEventListener("click", (e) => { + const clearRecent = e.target.closest("[data-clear-recent-items]"); + if (clearRecent) { + e.preventDefault(); + this.clearRecentItems(); + return; + } + + const item = e.target.closest(".result-item"); + if (item) { + const index = parseInt(item.dataset.index); + this.selectItem(index); + } + }); + + // Close on backdrop click + dialog.addEventListener("click", (e) => { + if (e.target === dialog) { + this.closeMenu(); + } + }); + + dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this.closeMenu(); + }); + + dialog.addEventListener("close", () => { + this.onMenuClosed(); + }); + + // Initial load + this.loadInitialData(); + } + + isInputFocused() { + const activeElement = document.activeElement; + return ( + activeElement && + (activeElement.tagName === "INPUT" || + activeElement.tagName === "TEXTAREA" || + activeElement.isContentEditable) + ); + } + + setElementAttribute(element, name, value) { + if (!element) { + return; + } + if (typeof element.setAttribute === "function") { + element.setAttribute(name, value); + } else { + element[name] = String(value); + } + } + + removeElementAttribute(element, name) { + if (!element) { + return; + } + if (typeof element.removeAttribute === "function") { + element.removeAttribute(name); + } else { + delete element[name]; + } + } + + focusRestoreTarget(trigger) { + if (trigger && typeof trigger.focus === "function") { + return trigger; + } + if ( + document.activeElement && + typeof document.activeElement.focus === "function" + ) { + return document.activeElement; + } + return null; + } + + setNavigationTriggersExpanded(expanded) { + if (typeof document.querySelectorAll !== "function") { + return; + } + document + .querySelectorAll("[data-navigation-search-open]") + .forEach((trigger) => { + this.setElementAttribute( + trigger, + "aria-expanded", + expanded ? "true" : "false", + ); + }); + } + + resultOptionId(index) { + return `${this.listboxId}-option-${index}`; + } + + updateComboboxState() { + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); + const matches = this.renderedMatches || []; + this.setElementAttribute( + input, + "aria-expanded", + dialog && dialog.open && matches.length > 0 ? "true" : "false", + ); + + if ( + dialog && + dialog.open && + this.selectedIndex >= 0 && + this.selectedIndex < matches.length + ) { + this.setElementAttribute( + input, + "aria-activedescendant", + this.resultOptionId(this.selectedIndex), + ); + } else { + this.removeElementAttribute(input, "aria-activedescendant"); + } + } + + setStatus(message) { + const status = this.shadowRoot.querySelector(`#${this.statusId}`); + if (status) { + status.textContent = message || ""; + } + } + + resultsStatus(count, truncated) { + if (truncated) { + return "More than 100 results. Keep typing to narrow the list."; + } + if (count === 0) { + return "No results found."; + } + if (count === 1) { + return "1 result."; + } + return `${count} results.`; + } + + loadInitialData() { + const itemsAttr = this.getAttribute("items"); + if (itemsAttr) { + try { + this.allItems = JSON.parse(itemsAttr); + this.matches = this.allItems; + } catch (e) { + console.error("Failed to parse items attribute:", e); + this.allItems = []; + this.matches = []; + } + } + } + + handleSearch(query) { + clearTimeout(this.debounceTimer); + if (query.trim()) { + this.setStatus("Searching..."); + } else { + this.setStatus(""); + } + + this.debounceTimer = setTimeout(() => { + const url = this.getAttribute("url"); + + if (url) { + // Fetch from API + this.fetchResults(url, query); + } else { + // Filter local items + this.filterLocalItems(query); + } + }, 200); + } + + async fetchResults(url, query) { + try { + const searchUrl = `${url}?q=${encodeURIComponent(query)}`; + const response = await fetch(searchUrl); + const data = await response.json(); + this.matches = data.matches || []; + this.selectedIndex = this.matches.length > 0 ? 0 : -1; + this.renderResults(); + if (query.trim()) { + this.setStatus(this.resultsStatus(this.matches.length, data.truncated)); + } else { + this.setStatus(""); + } + } catch (e) { + console.error("Failed to fetch search results:", e); + this.matches = []; + this.renderResults(); + this.setStatus("Search failed."); + } + } + + filterLocalItems(query) { + if (!query.trim()) { + this.matches = this.allItems || []; + } else { + const lowerQuery = query.toLowerCase(); + this.matches = (this.allItems || []).filter( + (item) => + item.name.toLowerCase().includes(lowerQuery) || + (item.display_name || "").toLowerCase().includes(lowerQuery) || + item.url.toLowerCase().includes(lowerQuery), + ); + } + this.selectedIndex = this.matches.length > 0 ? 0 : -1; + this.renderResults(); + if (query.trim()) { + this.setStatus(this.resultsStatus(this.matches.length, false)); + } else { + this.setStatus(""); + } + } + + recentItemsStorageKey() { + return "datasette.navigationSearch.recentItems"; + } + + loadRecentItems() { + if (typeof localStorage === "undefined") { + return []; + } + + try { + const raw = localStorage.getItem(this.recentItemsStorageKey()); + if (!raw) { + return []; + } + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) { + return []; + } + return parsed + .filter((item) => item && item.name && item.url) + .map((item) => ({ + name: String(item.name), + display_name: item.display_name ? String(item.display_name) : "", + url: String(item.url), + type: item.type ? String(item.type) : "", + description: item.description ? String(item.description) : "", + })) + .slice(0, 5); + } catch (e) { + return []; + } + } + + saveRecentItem(match) { + if ( + typeof localStorage === "undefined" || + !match || + !match.name || + !match.url + ) { + return; + } + + try { + const item = { + name: String(match.name), + display_name: match.display_name ? String(match.display_name) : "", + url: String(match.url), + type: match.type ? String(match.type) : "", + description: match.description ? String(match.description) : "", + }; + const recentItems = this.loadRecentItems().filter( + (recentItem) => recentItem.url !== item.url, + ); + localStorage.setItem( + this.recentItemsStorageKey(), + JSON.stringify([item, ...recentItems].slice(0, 5)), + ); + } catch (e) { + // localStorage may be unavailable, full, or disabled. + } + } + + clearRecentItems() { + if (typeof localStorage === "undefined") { + return; + } + + try { + localStorage.removeItem(this.recentItemsStorageKey()); + } catch (e) { + localStorage.setItem(this.recentItemsStorageKey(), "[]"); + } + this.renderResults(); + this.setStatus("Recent items cleared."); + } + + jumpSections() { + const manager = window.__DATASETTE__; + if (!manager || typeof manager.makeJumpSections !== "function") { + return []; + } + const sections = manager.makeJumpSections({ + navigationSearch: this, + }); + return Array.isArray(sections) + ? sections.filter( + (section) => section && typeof section.render === "function", + ) + : []; + } + + jumpSectionsHtml(jumpSections) { + return jumpSections + .map((section, index) => { + const id = section.id + ? ` data-jump-section-id="${this.escapeHtml(section.id)}"` + : ""; + return `
    `; + }) + .join(""); + } + + renderJumpSections(container, jumpSections) { + jumpSections.forEach((section, index) => { + const node = container.querySelector( + `[data-jump-section-index="${index}"]`, + ); + if (!node) { + return; + } + section.render(node, { + navigationSearch: this, + container, + input: this.shadowRoot.querySelector(".search-input"), + }); + }); + } + + resultItemHtml(match, index) { + const displayName = match.display_name || match.name; + const label = + match.display_name && match.display_name !== match.name + ? `
    ${this.escapeHtml(match.name)}
    ` + : ""; + const type = match.type + ? `
    ${this.escapeHtml(match.type)}
    ` + : ""; + const description = match.description + ? `
    ${this.escapeHtml( + match.description, + )}
    ` + : ""; + return ` +
    +
    + ${type} +
    ${this.escapeHtml(displayName)}
    + ${label} +
    ${this.escapeHtml(match.url)}
    + ${description} +
    +
    + `; + } + + renderResults() { + const container = this.shadowRoot.querySelector(".results-container"); + const input = this.shadowRoot.querySelector(".search-input"); + const showStartContent = !input.value.trim(); + const jumpSections = showStartContent ? this.jumpSections() : []; + const startBlock = showStartContent + ? this.jumpSectionsHtml(jumpSections) + : ""; + const recentItems = showStartContent ? this.loadRecentItems() : []; + const defaultMatches = showStartContent ? [] : this.matches; + const renderedMatches = [...recentItems, ...defaultMatches]; + this.renderedMatches = renderedMatches; + const emptyListbox = `
    `; + + if (renderedMatches.length) { + if ( + this.selectedIndex < 0 || + this.selectedIndex >= renderedMatches.length + ) { + this.selectedIndex = 0; + } + } else { + this.selectedIndex = -1; + } + + if (renderedMatches.length === 0) { + if (startBlock) { + container.innerHTML = startBlock + emptyListbox; + this.renderJumpSections(container, jumpSections); + } else if (showStartContent) { + container.innerHTML = emptyListbox; + } else { + const message = input.value.trim() + ? "No results found" + : "Start typing to search..."; + container.innerHTML = `${emptyListbox}
    ${message}
    `; + } + this.updateComboboxState(); + return; + } + + const recentHeading = recentItems.length + ? `
    Recent
    ` + : ""; + const recentGroup = recentItems.length + ? `
    ${recentItems + .map((match, index) => this.resultItemHtml(match, index)) + .join("")}
    ` + : ""; + const recentActions = recentItems.length + ? `
    ` + : ""; + const defaultHtml = defaultMatches + .map((match, index) => + this.resultItemHtml(match, recentItems.length + index), + ) + .join(""); + container.innerHTML = + startBlock + + recentHeading + + `
    ${recentGroup}${defaultHtml}
    ` + + recentActions; + this.renderJumpSections(container, jumpSections); + this.updateComboboxState(); + + // Scroll selected item into view + if (this.selectedIndex >= 0) { + const selectedItem = container.querySelector( + `.result-item[data-index="${this.selectedIndex}"]`, + ); + if (selectedItem) { + selectedItem.scrollIntoView({ block: "nearest" }); + } + } + } + + moveSelection(direction) { + const matches = this.renderedMatches || this.matches; + const newIndex = this.selectedIndex + direction; + if (newIndex >= 0 && newIndex < matches.length) { + this.selectedIndex = newIndex; + this.renderResults(); + } + } + + selectCurrentItem() { + const matches = this.renderedMatches || this.matches; + if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) { + this.selectItem(this.selectedIndex); + } + } + + selectItem(index) { + const matches = this.renderedMatches || this.matches; + const match = matches[index]; + if (match) { + this.saveRecentItem(match); + + // Dispatch custom event + this.dispatchEvent( + new CustomEvent("select", { + detail: match, + bubbles: true, + composed: true, + }), + ); + + // Navigate to URL + window.location.href = match.url; + + this.closeMenu({ restoreFocus: false }); + } + } + + openMenu(trigger) { + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); + + this.restoreFocusTarget = this.focusRestoreTarget(trigger); + this.shouldRestoreFocus = true; + if (!dialog.open) { + dialog.showModal(); + } + this.setNavigationTriggersExpanded(true); + input.value = ""; + input.focus(); + + // Reset state, then populate the default jump list. + this.matches = []; + this.selectedIndex = -1; + this.renderResults(); + this.setStatus(""); + } + + closeMenu(options = {}) { + const dialog = this.shadowRoot.querySelector("dialog"); + this.shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + this.onMenuClosed(); + } + } + + onMenuClosed() { + const input = this.shadowRoot.querySelector(".search-input"); + this.setElementAttribute(input, "aria-expanded", "false"); + this.removeElementAttribute(input, "aria-activedescendant"); + this.setNavigationTriggersExpanded(false); + this.setStatus(""); + if ( + this.shouldRestoreFocus && + this.restoreFocusTarget && + typeof this.restoreFocusTarget.focus === "function" + ) { + this.restoreFocusTarget.focus(); + } + this.restoreFocusTarget = null; + } + + escapeHtml(text) { + const div = document.createElement("div"); + div.textContent = text == null ? "" : text; + return div.innerHTML; + } +} + +// Register the custom element +customElements.define("navigation-search", NavigationSearch); diff --git a/datasette/static/table.js b/datasette/static/table.js index 909eebf3..e9115453 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -1,13 +1,6 @@ var DROPDOWN_HTML = `
    - +

      `; @@ -17,54 +10,509 @@ var DROPDOWN_ICON_SVG = ` `; +var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog"; +var setColumnTypeDialogState = null; + +function getParams() { + return new URLSearchParams(location.search); +} + +function paramsToUrl(params) { + var s = params.toString(); + return s ? "?" + s : location.pathname; +} + +function sortDescUrl(column) { + var params = getParams(); + params.set("_sort_desc", column); + params.delete("_sort"); + params.delete("_next"); + return paramsToUrl(params); +} + +function sortAscUrl(column) { + var params = getParams(); + params.set("_sort", column); + params.delete("_sort_desc"); + params.delete("_next"); + return paramsToUrl(params); +} + +function facetUrl(column) { + var params = getParams(); + params.append("_facet", column); + return paramsToUrl(params); +} + +function hideColumnUrl(column) { + var params = getParams(); + params.append("_nocol", column); + return paramsToUrl(params); +} + +function showAllColumnsUrl() { + var params = getParams(); + params.delete("_nocol"); + params.delete("_col"); + return paramsToUrl(params); +} + +function notBlankUrl(column) { + var params = getParams(); + params.set(`${column}__notblank`, "1"); + return paramsToUrl(params); +} + +function getDisplayedFacets() { + return Array.from(document.querySelectorAll(".facet-info")).map( + (el) => el.dataset.column, + ); +} + +function getColumnClassName(th) { + return Array.from(th.classList).find((className) => + className.startsWith("col-"), + ); +} + +function getColumnCells(th) { + var table = th.closest("table"); + var columnClassName = getColumnClassName(th); + if (!table || !columnClassName) { + return []; + } + return Array.from(table.querySelectorAll("td." + columnClassName)); +} + +function getColumnMeta(th) { + return { + columnName: th.dataset.column, + columnNotNull: th.dataset.columnNotNull === "1", + columnType: th.dataset.columnType, + isPk: th.dataset.isPk === "1", + }; +} + +function getColumnTypeText(th) { + var columnType = th.dataset.columnType; + if (!columnType) { + return null; + } + var notNull = th.dataset.columnNotNull === "1" ? " NOT NULL" : ""; + return `Type: ${columnType.toUpperCase()}${notNull}`; +} + +function getSetColumnTypeData() { + return window._setColumnTypeData || null; +} + +function getSetColumnTypeConfig(column) { + var data = getSetColumnTypeData(); + if (!data || !data.columns) { + return null; + } + return data.columns[column] || null; +} + +function canSetColumnType() { + return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch); +} + +function setColumnTypeActionLabel(column) { + var columnConfig = getSetColumnTypeConfig(column); + if (!columnConfig) { + return null; + } + return columnConfig.current + ? `Custom type: ${columnConfig.current.type}` + : "Set custom type"; +} + +function createSetColumnTypeOption(value, name, description, checked) { + var label = document.createElement("label"); + label.className = "set-column-type-option"; + + var input = document.createElement("input"); + input.type = "radio"; + input.name = "set-column-type-choice"; + input.value = value; + input.checked = checked; + + var content = document.createElement("span"); + content.className = "set-column-type-option-content"; + + var title = document.createElement("span"); + title.className = "set-column-type-option-name"; + title.textContent = name; + + var detail = document.createElement("span"); + detail.className = "set-column-type-option-description"; + detail.textContent = description; + + content.appendChild(title); + content.appendChild(detail); + label.appendChild(input); + label.appendChild(content); + return label; +} + +function setSetColumnTypeDialogBusy(state, isBusy) { + state.isBusy = isBusy; + state.saveButton.disabled = isBusy; + state.cancelButton.disabled = isBusy; + Array.from( + state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'), + ).forEach(function (input) { + input.disabled = isBusy; + }); + state.saveButton.textContent = isBusy ? "Saving..." : "Save"; +} + +function clearSetColumnTypeDialogError(state) { + state.error.hidden = true; + state.error.textContent = ""; +} + +function showSetColumnTypeDialogError(state, message) { + state.error.hidden = false; + state.error.textContent = message; +} + +function ensureSetColumnTypeDialog() { + if (setColumnTypeDialogState) { + return setColumnTypeDialogState; + } + if (!window.HTMLDialogElement) { + return null; + } + + var dialog = document.createElement("dialog"); + dialog.id = SET_COLUMN_TYPE_DIALOG_ID; + dialog.className = "set-column-type-dialog"; + dialog.setAttribute("aria-labelledby", "set-column-type-title"); + dialog.innerHTML = ` +
      + Set custom type + +
      +

      + +
      +
      + + + +
      + `; + document.body.appendChild(dialog); + + setColumnTypeDialogState = { + dialog: dialog, + meta: dialog.querySelector(".modal-meta"), + status: dialog.querySelector(".set-column-type-status"), + error: dialog.querySelector(".set-column-type-error"), + optionsWrap: dialog.querySelector(".set-column-type-options"), + footerInfo: dialog.querySelector(".footer-info"), + cancelButton: dialog.querySelector(".set-column-type-cancel"), + saveButton: dialog.querySelector(".set-column-type-save"), + currentColumn: null, + currentConfig: null, + isBusy: false, + }; + + setColumnTypeDialogState.cancelButton.addEventListener("click", function () { + if (!setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + if (setColumnTypeDialogState.isBusy) { + ev.preventDefault(); + } + }); + + dialog.addEventListener("close", function () { + clearSetColumnTypeDialogError(setColumnTypeDialogState); + setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); + }); + + setColumnTypeDialogState.saveButton.addEventListener("click", async function () { + var state = setColumnTypeDialogState; + var selected = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + var selectedType = selected ? selected.value : ""; + var currentType = state.currentConfig.current + ? state.currentConfig.current.type + : ""; + + if (selectedType === currentType) { + state.dialog.close(); + return; + } + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, true); + + var payload = { + column: state.currentColumn, + column_type: selectedType ? { type: selectedType } : null, + }; + + try { + var response = await fetch(getSetColumnTypeData().path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + var message = (data.errors || ["Request failed"]).join(" "); + throw new Error(message); + } + location.reload(); + } catch (error) { + setSetColumnTypeDialogBusy(state, false); + showSetColumnTypeDialogError(state, error.message || "Request failed"); + } + }); + + return setColumnTypeDialogState; +} + +function openSetColumnTypeDialog(th) { + var column = th.dataset.column; + var columnConfig = getSetColumnTypeConfig(column); + if (!columnConfig) { + return; + } + + var state = ensureSetColumnTypeDialog(); + if (!state) { + return; + } + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, false); + state.currentColumn = column; + state.currentConfig = columnConfig; + state.status.textContent = `Column: ${column}`; + state.meta.textContent = getColumnTypeText(th) || "Type unavailable"; + state.footerInfo.textContent = columnConfig.current + ? `Current custom type: ${columnConfig.current.type}` + : "No custom type set."; + state.optionsWrap.innerHTML = ""; + + var currentType = columnConfig.current ? columnConfig.current.type : ""; + state.optionsWrap.appendChild( + createSetColumnTypeOption( + "", + "No custom type", + "Use standard Datasette rendering without a custom type.", + currentType === "", + ), + ); + + columnConfig.options.forEach(function (option) { + state.optionsWrap.appendChild( + createSetColumnTypeOption( + option.name, + option.name, + option.description, + option.name === currentType, + ), + ); + }); + + if (!columnConfig.options.length) { + var emptyState = document.createElement("p"); + emptyState.className = "set-column-type-empty"; + emptyState.textContent = + "No registered custom types are compatible with this SQLite type."; + state.optionsWrap.appendChild(emptyState); + } + + if (!state.dialog.open) { + state.dialog.showModal(); + } + var selectedOption = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + if (selectedOption) { + selectedOption.focus(); + } else { + state.saveButton.focus(); + } +} + +function canChooseColumns() { + return !!( + document.querySelector("column-chooser") && window._columnChooserData + ); +} + +function shouldShowShowAllColumns() { + var params = getParams(); + return params.getAll("_nocol").length || params.getAll("_col").length; +} + +function hasMultipleVisibleColumns(manager) { + return ( + Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( + (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", + ).length > 1 + ); +} + +function buildColumnActionItems(manager, th, options) { + options = options || {}; + var params = getParams(); + var column = th.dataset.column; + var columnActions = []; + var isSortable = !!th.querySelector("a"); + var isFirstColumn = th.parentElement.querySelector("th:first-of-type") === th; + var isSinglePk = + th.dataset.isPk === "1" && + document.querySelectorAll('th[data-is-pk="1"]').length === 1; + var hasBlankValues = getColumnCells(th).some( + (el) => el.innerText.trim() === "", + ); + + if (isSortable && params.get("_sort") !== column) { + columnActions.push({ + label: "Sort ascending", + href: sortAscUrl(column), + }); + } + + if (isSortable && params.get("_sort_desc") !== column) { + columnActions.push({ + label: "Sort descending", + href: sortDescUrl(column), + }); + } + + if ( + DATASETTE_ALLOW_FACET && + !isFirstColumn && + !getDisplayedFacets().includes(column) && + !isSinglePk + ) { + columnActions.push({ + label: "Facet by this", + href: facetUrl(column), + }); + } + + if (options.includeChooseColumns && canChooseColumns()) { + columnActions.push({ + label: "Choose columns", + href: "#", + onClick: + options.onChooseColumns || + function (ev) { + ev.preventDefault(); + openColumnChooser(); + }, + }); + } + + if (canSetColumnType() && getSetColumnTypeConfig(column)) { + columnActions.push({ + label: setColumnTypeActionLabel(column), + href: "#", + onClick: + options.onSetColumnType || + function (ev) { + ev.preventDefault(); + window.setTimeout(function () { + openSetColumnTypeDialog(th); + }, 0); + }, + }); + } + + if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) { + columnActions.push({ + label: "Hide this column", + href: hideColumnUrl(column), + }); + } + + if (options.includeShowAllColumns && shouldShowShowAllColumns()) { + columnActions.push({ + label: "Show all columns", + href: showAllColumnsUrl(), + }); + } + + if (params.get(`${column}__notblank`) !== "1" && hasBlankValues) { + columnActions.push({ + label: "Show not-blank rows", + href: notBlankUrl(column), + }); + } + + return columnActions.concat(manager.makeColumnActions(getColumnMeta(th))); +} + +function buildColumnActionState(manager, th, options) { + return { + column: th.dataset.column, + columnDescription: th.dataset.columnDescription || null, + columnMeta: getColumnMeta(th), + columnTypeText: getColumnTypeText(th), + actionItems: buildColumnActionItems(manager, th, options), + }; +} + +function initializeColumnActions(manager) { + manager.columnActions = { + buildColumnActionState: function (th, options) { + return buildColumnActionState(manager, th, options); + }, + buildColumnActionItems: function (th, options) { + return buildColumnActionItems(manager, th, options); + }, + canChooseColumns: canChooseColumns, + facetUrl: facetUrl, + getColumnMeta: getColumnMeta, + getColumnTypeText: getColumnTypeText, + hideColumnUrl: hideColumnUrl, + notBlankUrl: notBlankUrl, + shouldShowShowAllColumns: shouldShowShowAllColumns, + showAllColumnsUrl: showAllColumnsUrl, + sortAscUrl: sortAscUrl, + sortDescUrl: sortDescUrl, + }; +} + +function renderActionLink(itemConfig) { + var newLink = document.createElement("a"); + newLink.textContent = itemConfig.label; + newLink.href = itemConfig.href || "#"; + if (itemConfig.onClick) { + newLink.addEventListener("click", itemConfig.onClick); + } + return newLink; +} + /** Main initialization function for Datasette Table interactions */ const initDatasetteTable = function (manager) { // Feature detection if (!window.URLSearchParams) { return; } - function getParams() { - return new URLSearchParams(location.search); - } - function paramsToUrl(params) { - var s = params.toString(); - return s ? "?" + s : location.pathname; - } - function sortDescUrl(column) { - var params = getParams(); - params.set("_sort_desc", column); - params.delete("_sort"); - params.delete("_next"); - return paramsToUrl(params); - } - function sortAscUrl(column) { - var params = getParams(); - params.set("_sort", column); - params.delete("_sort_desc"); - params.delete("_next"); - return paramsToUrl(params); - } - function facetUrl(column) { - var params = getParams(); - params.append("_facet", column); - return paramsToUrl(params); - } - function hideColumnUrl(column) { - var params = getParams(); - params.append("_nocol", column); - return paramsToUrl(params); - } - function showAllColumnsUrl() { - var params = getParams(); - params.delete("_nocol"); - params.delete("_col"); - return paramsToUrl(params); - } - function notBlankUrl(column) { - var params = getParams(); - params.set(`${column}__notblank`, "1"); - return paramsToUrl(params); - } function closeMenu() { menu.style.display = "none"; menu.classList.remove("anim-scale-in"); @@ -96,87 +544,41 @@ const initDatasetteTable = function (manager) { var rect = th.getBoundingClientRect(); var menuTop = rect.bottom + window.scrollY; var menuLeft = rect.left + window.scrollX; - var column = th.getAttribute("data-column"); - var params = getParams(); - var sort = menu.querySelector("a.dropdown-sort-asc"); - var sortDesc = menu.querySelector("a.dropdown-sort-desc"); - var facetItem = menu.querySelector("a.dropdown-facet"); - var notBlank = menu.querySelector("a.dropdown-not-blank"); - var hideColumn = menu.querySelector("a.dropdown-hide-column"); - var showAllColumns = menu.querySelector("a.dropdown-show-all-columns"); - if (params.get("_sort") == column) { - sort.parentNode.style.display = "none"; - } else { - sort.parentNode.style.display = "block"; - sort.setAttribute("href", sortAscUrl(column)); - } - if (params.get("_sort_desc") == column) { - sortDesc.parentNode.style.display = "none"; - } else { - sortDesc.parentNode.style.display = "block"; - sortDesc.setAttribute("href", sortDescUrl(column)); - } - /* Show hide columns options */ - if (params.get("_nocol") || params.get("_col")) { - showAllColumns.parentNode.style.display = "block"; - showAllColumns.setAttribute("href", showAllColumnsUrl()); - } else { - showAllColumns.parentNode.style.display = "none"; - } - if (th.getAttribute("data-is-pk") != "1") { - hideColumn.parentNode.style.display = "block"; - hideColumn.setAttribute("href", hideColumnUrl(column)); - } else { - hideColumn.parentNode.style.display = "none"; - } - /* Only show "Facet by this" if it's not the first column, not selected, - not a single PK and the Datasette allow_facet setting is True */ - var displayedFacets = Array.from( - document.querySelectorAll(".facet-info") - ).map((el) => el.dataset.column); - var isFirstColumn = - th.parentElement.querySelector("th:first-of-type") == th; - var isSinglePk = - th.getAttribute("data-is-pk") == "1" && - document.querySelectorAll('th[data-is-pk="1"]').length == 1; - if ( - !DATASETTE_ALLOW_FACET || - isFirstColumn || - displayedFacets.includes(column) || - isSinglePk - ) { - facetItem.parentNode.style.display = "none"; - } else { - facetItem.parentNode.style.display = "block"; - facetItem.setAttribute("href", facetUrl(column)); - } - /* Show notBlank option if not selected AND at least one visible blank value */ - var tdsForThisColumn = Array.from( - th.closest("table").querySelectorAll("td." + th.className) - ); - if ( - params.get(`${column}__notblank`) != "1" && - tdsForThisColumn.filter((el) => el.innerText.trim() == "").length - ) { - notBlank.parentNode.style.display = "block"; - notBlank.setAttribute("href", notBlankUrl(column)); - } else { - notBlank.parentNode.style.display = "none"; - } - var columnTypeP = menu.querySelector(".dropdown-column-type"); - var columnType = th.dataset.columnType; - var notNull = th.dataset.columnNotNull == 1 ? " NOT NULL" : ""; + var actionState = manager.columnActions.buildColumnActionState(th, { + includeChooseColumns: true, + includeShowAllColumns: true, + onChooseColumns: function (ev) { + ev.preventDefault(); + closeMenu(); + openColumnChooser(); + }, + onSetColumnType: function (ev) { + ev.preventDefault(); + closeMenu(); + window.setTimeout(function () { + openSetColumnTypeDialog(th); + }, 0); + }, + }); + var menuList = menu.querySelector("ul.dropdown-actions"); + menuList.innerHTML = ""; + actionState.actionItems.forEach((itemConfig) => { + var menuItem = document.createElement("li"); + menuItem.appendChild(renderActionLink(itemConfig)); + menuList.appendChild(menuItem); + }); - if (columnType) { + var columnTypeP = menu.querySelector(".dropdown-column-type"); + if (actionState.columnTypeText) { columnTypeP.style.display = "block"; - columnTypeP.innerText = `Type: ${columnType.toUpperCase()}${notNull}`; + columnTypeP.innerText = actionState.columnTypeText; } else { columnTypeP.style.display = "none"; } var columnDescriptionP = menu.querySelector(".dropdown-column-description"); - if (th.dataset.columnDescription) { - columnDescriptionP.innerText = th.dataset.columnDescription; + if (actionState.columnDescription) { + columnDescriptionP.innerText = actionState.columnDescription; columnDescriptionP.style.display = "block"; } else { columnDescriptionP.style.display = "none"; @@ -187,37 +589,6 @@ const initDatasetteTable = function (manager) { menu.style.display = "block"; menu.classList.add("anim-scale-in"); - // Custom menu items on each render - // Plugin hook: allow adding JS-based additional menu items - const columnActionsPayload = { - columnName: th.dataset.column, - columnNotNull: th.dataset.columnNotNull === '1', - columnType: th.dataset.columnType, - isPk: th.dataset.isPk === '1' - }; - const columnItemConfigs = manager.makeColumnActions(columnActionsPayload); - - const menuList = menu.querySelector('ul'); - columnItemConfigs.forEach(itemConfig => { - // Remove items from previous render. We assume entries have unique labels. - const existingItems = menuList.querySelectorAll(`li`); - Array.from(existingItems).filter(item => item.innerText === itemConfig.label).forEach(node => { - node.remove(); - }); - - const newLink = document.createElement('a'); - newLink.textContent = itemConfig.label; - newLink.href = itemConfig.href ?? '#'; - if (itemConfig.onClick) { - newLink.onclick = itemConfig.onClick; - } - - // Attach new elements to DOM - const menuItem = document.createElement('li'); - menuItem.appendChild(newLink); - menuList.appendChild(menuItem); - }); - // Measure width of menu and adjust position if too far right const menuWidth = menu.offsetWidth; const windowWidth = window.innerWidth; @@ -225,17 +596,17 @@ const initDatasetteTable = function (manager) { menu.style.left = windowWidth - menuWidth - 20 + "px"; } // Align menu .hook arrow with the column cog icon - const hook = menu.querySelector('.hook'); - const icon = th.querySelector('.dropdown-menu-icon'); + const hook = menu.querySelector(".hook"); + const icon = th.querySelector(".dropdown-menu-icon"); const iconRect = icon.getBoundingClientRect(); - const hookLeft = (iconRect.left - menuLeft + 1) + 'px'; + const hookLeft = iconRect.left - menuLeft + 1 + "px"; hook.style.left = hookLeft; // Move the whole menu right if the hook is too far right const menuRect = menu.getBoundingClientRect(); if (iconRect.right > menuRect.right) { - menu.style.left = (iconRect.right - menuWidth) + 'px'; + menu.style.left = iconRect.right - menuWidth + "px"; // And move hook tip as well - hook.style.left = (menuWidth - 13) + 'px'; + hook.style.left = menuWidth - 13 + "px"; } } @@ -250,7 +621,9 @@ const initDatasetteTable = function (manager) { menu.style.display = "none"; document.body.appendChild(menu); - var ths = Array.from(document.querySelectorAll(manager.selectors.tableHeaders)); + var ths = Array.from( + document.querySelectorAll(manager.selectors.tableHeaders), + ); ths.forEach((th) => { if (!th.querySelector("a")) { return; @@ -264,9 +637,9 @@ const initDatasetteTable = function (manager) { /* Add x buttons to the filter rows */ function addButtonsToFilterRows(manager) { var x = "✖"; - var rows = Array.from(document.querySelectorAll(manager.selectors.filterRow)).filter((el) => - el.querySelector(".filter-op") - ); + var rows = Array.from( + document.querySelectorAll(manager.selectors.filterRow), + ).filter((el) => el.querySelector(".filter-op")); rows.forEach((row) => { var a = document.createElement("a"); a.setAttribute("href", "#"); @@ -287,18 +660,18 @@ function addButtonsToFilterRows(manager) { a.style.display = "none"; } }); -}; +} /* Set up datalist autocomplete for filter values */ function initAutocompleteForFilterValues(manager) { function createDataLists() { var facetResults = document.querySelectorAll( - manager.selectors.facetResults + manager.selectors.facetResults, ); Array.from(facetResults).forEach(function (facetResult) { // Use link text from all links in the facet result var links = Array.from( - facetResult.querySelectorAll("li:not(.facet-truncated) a") + facetResult.querySelectorAll("li:not(.facet-truncated) a"), ); // Create a datalist element var datalist = document.createElement("datalist"); @@ -324,12 +697,57 @@ function initAutocompleteForFilterValues(manager) { .setAttribute("list", "datalist-" + event.target.value); } }); -}; +} + +/** Open the column-chooser web component */ +function openColumnChooser() { + var chooser = document.querySelector("column-chooser"); + var data = window._columnChooserData; + if (!chooser || !data) return; + + var nonPkColumns = data.allColumns.filter(function (col) { + return data.primaryKeys.indexOf(col) === -1; + }); + var selected = data.selectedColumns.filter(function (col) { + return data.primaryKeys.indexOf(col) === -1; + }); + + chooser.open({ + columns: nonPkColumns, + selected: selected, + onApply: function (cols) { + var params = new URLSearchParams(location.search); + params.delete("_col"); + params.delete("_nocol"); + params.delete("_next"); + + if (cols.length === nonPkColumns.length) { + // Check if order matches original - if so, no params needed + var orderMatches = cols.every(function (col, i) { + return col === nonPkColumns[i]; + }); + if (!orderMatches) { + cols.forEach(function (col) { + params.append("_col", col); + }); + } + } else { + cols.forEach(function (col) { + params.append("_col", col); + }); + } + var qs = params.toString(); + location.href = qs ? "?" + qs : location.pathname; + }, + }); +} // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; + initializeColumnActions(manager); + // Main table initDatasetteTable(manager); diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py new file mode 100644 index 00000000..a6123daa --- /dev/null +++ b/datasette/stored_queries.py @@ -0,0 +1,581 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any, Iterable + +from .utils import tilde_encode, urlsafe_components + +UNCHANGED = object() + + +QUERY_OPTION_FIELDS = ( + "hide_sql", + "fragment", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +) + + +@dataclass +class StoredQuery: + database: str + name: str + sql: str + title: str | None + description: str | None + description_html: str | None + hide_sql: bool + fragment: str | None + parameters: list[str] + is_write: bool + is_private: bool + is_trusted: bool + source: str + owner_id: str | None + on_success_message: str | None + on_success_message_sql: str | None + on_success_redirect: str | None + on_error_message: str | None + on_error_redirect: str | None + private: bool | None = None + + +@dataclass +class StoredQueryPage: + queries: list[StoredQuery] + next: str | None + has_more: bool + limit: int + + +def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: + data = { + "database": query.database, + "name": query.name, + "sql": query.sql, + "title": query.title, + "description": query.description, + "description_html": query.description_html, + "hide_sql": query.hide_sql, + "fragment": query.fragment, + "params": list(query.parameters), + "parameters": list(query.parameters), + "is_write": query.is_write, + "is_private": query.is_private, + "is_trusted": query.is_trusted, + "source": query.source, + "owner_id": query.owner_id, + "on_success_message": query.on_success_message, + "on_success_message_sql": query.on_success_message_sql, + "on_success_redirect": query.on_success_redirect, + "on_error_message": query.on_error_message, + "on_error_redirect": query.on_error_redirect, + } + if query.private is not None: + data["private"] = query.private + return data + + +def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: + return { + "queries": [stored_query_to_dict(query) for query in page.queries], + "next": page.next, + "has_more": page.has_more, + "limit": page.limit, + } + + +async def save_queries_from_config(datasette: Any) -> None: + # Apply configured query entries from datasette.yaml to the internal table. + await datasette.get_internal_database().execute_write( + "DELETE FROM queries WHERE source = 'config'" + ) + for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if not isinstance(query_config, dict): + query_config = {"sql": query_config} + await datasette.add_query( + dbname, + query_name, + query_config["sql"], + title=query_config.get("title"), + description=query_config.get("description"), + description_html=query_config.get("description_html"), + hide_sql=bool(query_config.get("hide_sql")), + fragment=query_config.get("fragment"), + parameters=query_config.get("params"), + is_write=bool(query_config.get("write")), + is_trusted=bool(query_config.get("is_trusted", True)), + source="config", + on_success_message=query_config.get("on_success_message"), + on_success_message_sql=query_config.get("on_success_message_sql"), + on_success_redirect=query_config.get("on_success_redirect"), + on_error_message=query_config.get("on_error_message"), + on_error_redirect=query_config.get("on_error_redirect"), + ) + + +def query_row_to_stored_query( + row: Any, private: bool | None = None +) -> StoredQuery | None: + if row is None: + return None + parameters = json.loads(row["parameters"] or "[]") + options = json.loads(row["options"] or "{}") + return StoredQuery( + database=row["database_name"], + name=row["name"], + sql=row["sql"], + title=row["title"], + description=row["description"], + description_html=row["description_html"], + hide_sql=bool(options.get("hide_sql")), + fragment=options.get("fragment"), + parameters=parameters, + is_write=bool(row["is_write"]), + is_private=bool(row["is_private"]), + is_trusted=bool(row["is_trusted"]), + source=row["source"], + owner_id=row["owner_id"], + on_success_message=options.get("on_success_message"), + on_success_message_sql=options.get("on_success_message_sql"), + on_success_redirect=options.get("on_success_redirect"), + on_error_message=options.get("on_error_message"), + on_error_redirect=options.get("on_error_redirect"), + private=private, + ) + + +def query_options_json(options: dict[str, Any]) -> str: + options_dict = {} + for field in QUERY_OPTION_FIELDS: + value = options.get(field) + if field == "hide_sql": + if value: + options_dict[field] = True + elif value is not None: + options_dict[field] = value + return json.dumps(options_dict, sort_keys=True) + + +async def add_query( + datasette: Any, + database: str, + name: str, + sql: str, + *, + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, +) -> None: + parameters_json = json.dumps(list(parameters or [])) + options_json = query_options_json( + { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + ) + sql_statement = """ + INSERT INTO queries ( + database_name, name, sql, title, description, description_html, + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + if replace: + sql_statement += """ + ON CONFLICT(database_name, name) DO UPDATE SET + sql = excluded.sql, + title = excluded.title, + description = excluded.description, + description_html = excluded.description_html, + options = excluded.options, + parameters = excluded.parameters, + is_write = excluded.is_write, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, + source = excluded.source, + owner_id = excluded.owner_id, + updated_at = CURRENT_TIMESTAMP + """ + await datasette.get_internal_database().execute_write( + sql_statement, + [ + database, + name, + sql, + title, + description, + description_html, + options_json, + parameters_json, + int(bool(is_write)), + int(bool(is_private)), + int(bool(is_trusted)), + source, + owner_id, + ], + ) + + +async def update_query( + datasette: Any, + database: str, + name: str, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, +) -> None: + fields = { + "sql": sql, + "title": title, + "description": description, + "description_html": description_html, + "parameters": parameters, + "is_write": is_write, + "is_private": is_private, + "is_trusted": is_trusted, + "source": source, + "owner_id": owner_id, + } + option_fields = { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + updates = [] + params = [] + for field, value in fields.items(): + if value is UNCHANGED: + continue + if field in {"is_write", "is_private", "is_trusted"}: + value = int(bool(value)) + elif field == "parameters": + value = json.dumps(list(value or [])) + updates.append(f"{field} = ?") + params.append(value) + changed_options = { + field: value for field, value in option_fields.items() if value is not UNCHANGED + } + if changed_options: + rows = await datasette.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + row = rows.first() + options = json.loads(row["options"] or "{}") if row is not None else {} + for field, value in changed_options.items(): + if field == "hide_sql": + if value: + options[field] = True + else: + options.pop(field, None) + elif value is None: + options.pop(field, None) + else: + options[field] = value + updates.append("options = ?") + params.append(json.dumps(options, sort_keys=True)) + if not updates: + return + updates.append("updated_at = CURRENT_TIMESTAMP") + params.extend([database, name]) + await datasette.get_internal_database().execute_write( + """ + UPDATE queries + SET {} + WHERE database_name = ? AND name = ? + """.format(", ".join(updates)), + params, + ) + + +async def remove_query( + datasette: Any, database: str, name: str, source: str | None = None +) -> None: + sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" + params = [database, name] + if source is not None: + sql += " AND source = ?" + params.append(source) + await datasette.get_internal_database().execute_write(sql, params) + + +async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: + rows = await datasette.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + return query_row_to_stored_query(rows.first()) + + +async def count_queries( + datasette: Any, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, +) -> int: + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await datasette.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + + +async def list_queries( + datasette: Any, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, +) -> StoredQueryPage: + limit = min(max(1, int(limit)), 1000) + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + include_is_private=include_private, + ) + params = dict(allowed_params) + params.update({"limit": limit + 1}) + sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" + + if cursor: + try: + components = urlsafe_components(cursor) + except ValueError: + components = [] + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: + where_clauses.append(""" + ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_sort_key"] = components[0] + params["cursor_name"] = components[1] + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + private_select = ", allowed.is_private AS private" if include_private else "" + rows = list( + ( + await datasette.get_internal_database().execute( + """ + SELECT q.*, {sort_key_sql} AS sort_key{private_select} + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + ORDER BY {order_by} + LIMIT :limit + """.format( + allowed_sql=allowed_sql, + private_select=private_select, + sort_key_sql=sort_key_sql, + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, + ), + params, + ) + ).rows + ) + has_more = len(rows) > limit + if has_more: + rows = rows[:limit] + + queries = [] + for row in rows: + query = query_row_to_stored_query( + row, private=bool(row["private"]) if include_private else None + ) + assert query is not None + queries.append(query) + + next_token = None + if has_more and rows: + last_row = rows[-1] + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + return StoredQueryPage( + queries=queries, + next=next_token, + has_more=has_more, + limit=limit, + ) diff --git a/datasette/templates/_action_menu.html b/datasette/templates/_action_menu.html index 7d1d4a55..1ae8c173 100644 --- a/datasette/templates/_action_menu.html +++ b/datasette/templates/_action_menu.html @@ -1,7 +1,7 @@ {% if action_links %}
      - +
      -
        +
          {% for link in action_links %} -
        • {{ link.label }} +
        • {{ link.label }} {% if link.description %}

          {{ link.description }}

          {% endif %}           diff --git a/datasette/templates/_close_open_menus.html b/datasette/templates/_close_open_menus.html index 3302d77d..a24633d0 100644 --- a/datasette/templates/_close_open_menus.html +++ b/datasette/templates/_close_open_menus.html @@ -13,4 +13,50 @@ document.body.addEventListener('click', (ev) => { (details) => details.open && details != detailsClickedWithin ).forEach(details => details.open = false); }); + +/* Sync aria-expanded and add keyboard navigation for details-menu elements */ +document.querySelectorAll('details.details-menu').forEach(function(details) { + var summary = details.querySelector('summary'); + details.addEventListener('toggle', function() { + if (summary) { + summary.setAttribute('aria-expanded', details.open ? 'true' : 'false'); + } + if (details.open) { + /* Focus first menu item when menu opens */ + var firstItem = details.querySelector('[role="menuitem"]'); + if (firstItem) { firstItem.focus(); } + } + }); +}); + +document.body.addEventListener('keydown', function(ev) { + /* Keyboard navigation for open details-menu elements */ + var openDetails = Array.from(document.querySelectorAll('details.details-menu[open]')); + if (!openDetails.length) { return; } + + if (ev.key === 'Escape') { + openDetails.forEach(function(details) { + details.open = false; + var summary = details.querySelector('summary'); + if (summary) { summary.focus(); } + }); + return; + } + + if (ev.key === 'ArrowDown' || ev.key === 'ArrowUp') { + var focused = document.activeElement; + openDetails.forEach(function(details) { + var items = Array.from(details.querySelectorAll('[role="menuitem"]')); + if (!items.length) { return; } + var idx = items.indexOf(focused); + if (idx === -1) { return; } + ev.preventDefault(); + if (ev.key === 'ArrowDown') { + items[(idx + 1) % items.length].focus(); + } else { + items[(idx - 1 + items.length) % items.length].focus(); + } + }); + } +}); diff --git a/datasette/templates/_debug_common_functions.html b/datasette/templates/_debug_common_functions.html new file mode 100644 index 00000000..d988a2f3 --- /dev/null +++ b/datasette/templates/_debug_common_functions.html @@ -0,0 +1,50 @@ + diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html new file mode 100644 index 00000000..a19bae13 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_scripts.html @@ -0,0 +1,111 @@ + diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html new file mode 100644 index 00000000..165cfe9f --- /dev/null +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -0,0 +1,41 @@ + diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html new file mode 100644 index 00000000..d7203c1e --- /dev/null +++ b/datasette/templates/_permissions_debug_tabs.html @@ -0,0 +1,54 @@ +{% if has_debug_permission %} +{% set query_string = '?' + request.query_string if request.query_string else '' %} + + + + +{% endif %} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html new file mode 100644 index 00000000..159a141c --- /dev/null +++ b/datasette/templates/_sql_parameter_scripts.html @@ -0,0 +1,293 @@ + diff --git a/datasette/templates/_sql_parameter_styles.html b/datasette/templates/_sql_parameter_styles.html new file mode 100644 index 00000000..bc6838f5 --- /dev/null +++ b/datasette/templates/_sql_parameter_styles.html @@ -0,0 +1,58 @@ + diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html new file mode 100644 index 00000000..58801d40 --- /dev/null +++ b/datasette/templates/_sql_parameters.html @@ -0,0 +1,9 @@ +
          + {% if parameter_names %} +

          Parameters

          + {% for parameter in parameter_names %} + {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} +

          {% if sql_parameters_allow_expand|default(false) %} {% endif %}

          + {% endfor %} + {% endif %} +
          diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index a1329ba7..f47a325f 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,12 +1,12 @@
          -{% if display_rows %} +{% if display_columns %}
      {% for column in display_columns %} -
      + {% if not column.sortable %} {{ column.name }} {% else %} @@ -31,6 +31,7 @@
      -{% else %} +{% endif %} +{% if not display_rows %}

      0 records

      {% endif %} diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 610417d2..1ecc92df 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -33,6 +33,9 @@ p.message-warning {

      Debug allow rules

      +{% set current_tab = "allow_debug" %} +{% include "_permissions_debug_tabs.html" %} +

      Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

      diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 0b2def5a..e1767deb 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -20,7 +20,7 @@