diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 00000000..e56d9c27 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b0640ae8..b6c58c7e 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -1,11 +1,10 @@ name: Deploy latest.datasette.io on: - workflow_dispatch: push: branches: - main - # - 1.0-dev + - cloudrun-fix permissions: contents: read @@ -15,16 +14,24 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v6 + # Using Python 3.10 for gcloud compatibility: with: - python-version: "3.13" - cache: pip + python-version: "3.10" + - uses: actions/cache@v4 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install . --group dev + python -m pip install -e .[test] + python -m pip install -e .[docs] python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests if: ${{ github.ref == 'refs/heads/main' }} @@ -57,7 +64,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable stored query demo + - name: And the counters writable canned query demo run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index b8fb8aaa..a54bd83a 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request: + pull_request_target: types: - opened diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 735e14e9..77cce7d1 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v6 - - uses: actions/cache@v5 + uses: actions/checkout@v4 + - uses: actions/cache@v4 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87300593..6c83346d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,16 +14,16 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: setup.py - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[test]' - name: Run tests run: | pytest @@ -35,13 +35,13 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.13' cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: setup.py - name: Install dependencies run: | pip install setuptools wheel build @@ -56,16 +56,16 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.10' cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: setup.py - name: Install dependencies run: | - python -m pip install . --group dev + python -m pip install -e .[docs] python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index e622ef4c..afe8d6b2 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 9a808194..8a47fd2d 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,16 +9,16 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.11' cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + cache-dependency-path: '**/setup.py' - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[docs]' - name: Check spelling run: | codespell README.md --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml deleted file mode 100644 index 59b5fbc0..00000000 --- a/.github/workflows/stable-docs.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Update Stable Docs - -on: - release: - types: [published] - push: - branches: - - main - -permissions: - contents: write - -jobs: - update_stable_docs: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 # We need all commits to find docs/ changes - - name: Set up Git user - run: | - git config user.name "Automated" - git config user.email "actions@users.noreply.github.com" - - name: Create stable branch if it does not yet exist - run: | - if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then - # Make sure we have all tags locally - git fetch --tags --quiet - - # Latest tag that is just numbers and dots (optionally prefixed with 'v') - # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. - LATEST_RELEASE=$( - git tag -l --sort=-v:refname \ - | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ - | head -n1 - ) - - git checkout -b stable - - # If there are any stable releases, copy docs/ from the most recent - if [ -n "$LATEST_RELEASE" ]; then - rm -rf docs/ - git checkout "$LATEST_RELEASE" -- docs/ || true - fi - - git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" - git push -u origin stable - fi - - name: Handle Release - if: github.event_name == 'release' && !github.event.release.prerelease - run: | - git fetch --all - git checkout stable - git reset --hard ${GITHUB_REF#refs/tags/} - git push origin stable --force - - name: Handle Commit to Main - if: contains(github.event.head_commit.message, '!stable-docs') - run: | - git fetch origin - git checkout -b stable origin/stable - # Get the list of modified files in docs/ from the current commit - FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) - # Check if the list of files is non-empty - if [[ -n "$FILES" ]]; then - # Checkout those files to the stable branch to over-write with their contents - for FILE in $FILES; do - git checkout ${{ github.sha }} -- $FILE - done - git add docs/ - git commit -m "Doc changes from ${{ github.sha }}" - git push origin stable - else - echo "No changes to docs/ in this commit." - exit 0 - fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index c514048e..22a69150 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,17 +15,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.12' cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + cache-dependency-path: '**/setup.py' - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install . --group dev + python -m pip install -e .[test] python -m pip install pytest-cov - name: Run tests run: |- diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5162c47a..7357b30c 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,15 +12,15 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: python-version: "3.10" cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + cache-dependency-path: '**/setup.py' - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v4 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 23fce459..698aec8a 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,14 +25,14 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: setup.py - name: Set up SQLite ${{ matrix.sqlite-version }} uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 with: @@ -45,7 +45,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[test]' pip freeze - name: Run tests run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1b2e9d2..901c4905 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,20 +12,20 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: setup.py - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[test]' pip freeze - name: Run tests run: | @@ -33,12 +33,9 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Black + - name: Install docs dependencies run: | - black --version - black --check . - - name: Ruff - run: ruff check datasette tests + pip install -e '.[docs]' - name: Check if cog needs to be run run: | cog --check docs/*.rst diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index a033cd92..fcee0f21 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 72af1eec..9792245d 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -5,14 +5,11 @@ on: permissions: contents: read - models: read jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 12acd87e..277ff653 100644 --- a/.gitignore +++ b/.gitignore @@ -5,12 +5,6 @@ scratchpad .vscode -uv.lock -data.db - -# test databases -*.db - # We don't use Pipfile, so ignore them Pipfile Pipfile.lock @@ -129,6 +123,4 @@ node_modules # include it in source control. tests/*.dylib tests/*.so -tests/*.dll - -.idea \ No newline at end of file +tests/*.dll \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b3e54aa..5b30e75a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,17 +1,16 @@ version: 2 -sphinx: - configuration: docs/conf.py - build: - os: ubuntu-24.04 + os: ubuntu-20.04 tools: - python: "3.13" - jobs: - install: - - pip install --upgrade pip - - pip install . --group dev + python: "3.11" -formats: -- pdf -- epub +sphinx: + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..e7497690 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.14-slim-trixie as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/Justfile b/Justfile index 657881be..172de444 100644 --- a/Justfile +++ b/Justfile @@ -5,56 +5,38 @@ export DATASETTE_SECRET := "not_a_secret" # Setup project @init: - uv sync + pipenv run pip install -e '.[test,docs]' # Run pytest with supplied options -@test *options: init - uv run pytest -n auto {{options}} +@test *options: + pipenv run pytest {{options}} @codespell: - uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt - uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - uv run codespell tests --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, ruff, cog +# Run linters: black, flake8, mypy, cog @lint: codespell - uv run black datasette tests --check - uv run ruff check datasette tests - uv run cog --check README.md docs/*.rst - -# Apply ruff fixes -@fix: - uv run ruff check --fix datasette tests + pipenv run black . --check + pipenv run flake8 + pipenv run cog --check README.md docs/*.rst # Rebuild docs with cog @cog: - uv run cog -r README.md docs/*.rst + pipenv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: cog blacken-docs - uv run make -C docs livehtml - -# Build docs as static HTML -@docs-build: cog blacken-docs - rm -rf docs/_build && cd docs && uv run make html +@docs: cog + pipenv run blacken-docs -l 60 docs/*.rst + cd docs && pipenv run make livehtml # Apply Black @black: - uv run black datasette tests + pipenv run black . -# Apply blacken-docs -@blacken-docs: - uv run blacken-docs -l 60 docs/*.rst - -# Apply prettier -@prettier: - npm run fix - -# Format code with both black and prettier -@format: black prettier blacken-docs - -@serve *options: - uv run sqlite-utils create-database data.db - uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore - uv run python -m datasette data.db --root --reload {{options}} +@serve: + pipenv run sqlite-utils create-database data.db + pipenv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + pipenv run python -m datasette data.db --root --reload diff --git a/datasette/__init__.py b/datasette/__init__.py index eb18e59e..47d2b4f6 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,7 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenRestrictions # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py deleted file mode 100644 index 5fb6b473..00000000 --- a/datasette/_pytest_plugin.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Pytest plugin that automatically closes any Datasette instances constructed -during a pytest test — both in the test body and in function-scoped -fixtures. Instances constructed by session-, module-, class- or package- -scoped fixtures are left alone, because other tests in the session will -still want to use them. - -Registered as a pytest11 entry point in pyproject.toml so that downstream -projects using Datasette get the same FD-safety net for their own tests. - -Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the -equivalent ini file). -""" - -from __future__ import annotations - -import contextvars -import weakref - -import pytest - -from datasette.app import Datasette - -_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( - "datasette_active_instances", default=None -) - -_original_init = Datasette.__init__ - - -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) - - -Datasette.__init__ = _tracking_init - - -def pytest_addoption(parser): - parser.addini( - "datasette_autoclose", - help=( - "Automatically close Datasette instances created inside test " - "bodies and function-scoped fixtures (default: true)." - ), - default="true", - ) - - -def _enabled(config) -> bool: - value = config.getini("datasette_autoclose") - if isinstance(value, bool): - return value - return str(value).strip().lower() not in ("false", "0", "no", "off") - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - """Track Datasette instances across setup, call and teardown; close at end.""" - if not _enabled(item.config): - yield - return - refs: list[weakref.ref] = [] - token = _active_instances.set(refs) - try: - yield - finally: - _active_instances.reset(token) - for ref in reversed(refs): - ds = ref() - if ds is None: - continue - try: - ds.close() - except Exception as e: - item.warn( - pytest.PytestUnraisableExceptionWarning( - f"Error closing Datasette instance: {e!r}" - ) - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef, request): - """Exempt instances created by non-function-scoped fixtures. - - Session-, module-, class- and package-scoped fixtures produce Datasette - instances that must survive beyond the current test — other tests in - the session will still use them. When such a fixture creates one or - more Datasette instances during its setup, we snapshot the tracking - list before the fixture runs and subtract off any instances that were - added during its setup, so they don't get closed at test teardown. - """ - refs = _active_instances.get() - if refs is None: - yield - return - before_ids = {id(ref) for ref in refs} - yield - if fixturedef.scope != "function": - new_refs = [ref for ref in refs if id(ref) not in before_ids] - for new_ref in new_refs: - try: - refs.remove(new_ref) - except ValueError: - pass diff --git a/datasette/app.py b/datasette/app.py index e7f34e69..6c7026a8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,12 +1,7 @@ -from __future__ import annotations - +from asgi_csrf import Errors import asyncio -import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List - -if TYPE_CHECKING: - from datasette.permissions import Resource - from datasette.tokens import TokenRestrictions +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union +import asgi_csrf import collections import dataclasses import datetime @@ -41,26 +36,8 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .events import Event -from .column_types import SQLiteType -from . import stored_queries, write_sql from .views import Context -from .views.database import ( - database_download, - DatabaseView, - TableCreateView, - QueryView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) +from .views.database import database_download, DatabaseView, TableCreateView, QueryView from .views.index import IndexView from .views.special import ( JsonDataView, @@ -75,15 +52,10 @@ from .views.special import ( AllowedResourcesView, PermissionRulesView, PermissionCheckView, - JumpView, - InstanceSchemaView, - DatabaseSchemaView, - TableSchemaView, ) from .views.table import ( TableInsertView, TableUpsertView, - TableSetColumnTypeView, TableDropView, table_view, ) @@ -93,7 +65,6 @@ from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( - PaginatedResources, PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, @@ -114,7 +85,6 @@ from .utils import ( resolve_env_secrets, resolve_routes, tilde_decode, - tilde_encode, to_css_class, urlsafe_components, redact_keys, @@ -135,7 +105,6 @@ from .utils.asgi import ( asgi_send_file, asgi_send_redirect, ) -from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -145,39 +114,10 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .resources import DatabaseResource, TableResource +from .utils.permissions import build_rules_union, PluginSQL app_root = Path(__file__).parent.parent - -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) - - -class _DatasetteClientContext: - """Context manager to mark code as executing within a datasette.client request.""" - - def __enter__(self): - self.token = _in_datasette_client.set(True) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - _in_datasette_client.reset(self.token) - return False - - -@dataclasses.dataclass -class PermissionCheck: - """Represents a logged permission check for debugging purposes.""" - - when: str - actor: Dict[str, Any] | None - action: str - parent: str | None - child: str | None - result: bool - - # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 SQLITE_LIMIT_ATTACHED = 10 @@ -287,9 +227,6 @@ FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() -ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) - - async def favicon(request, send): await asgi_send_file( send, @@ -340,10 +277,8 @@ class Datasette: crossdb=False, nolock=False, internal=None, - default_deny=False, ): self._startup_invoked = False - self._closed = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -372,8 +307,7 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() - self.actions = {} # .invoke_startup() will populate this - self._column_types = {} # .invoke_startup() will populate this + self.permissions = {} # .invoke_startup() will populate this try: self._refresh_schemas_lock = asyncio.Lock() except RuntimeError as rex: @@ -398,7 +332,7 @@ class Datasette: self.internal_db_created = False if internal is None: - self._internal_database = Database(self, is_temp_disk=True) + self._internal_database = Database(self, memory_name=secrets.token_hex()) else: self._internal_database = Database(self, path=internal, mode="rwc") self._internal_database.name = INTERNAL_DB_NAME @@ -457,37 +391,10 @@ class Datasette: config = config or {} config_settings = config.get("settings") or {} - # Validate settings from config file - for key, value in config_settings.items(): + # validate "settings" keys in datasette.json + for key in config_settings: if key not in DEFAULT_SETTINGS: - raise StartupError(f"Invalid setting '{key}' in config file") - # Validate type matches expected type from DEFAULT_SETTINGS - if value is not None: # Allow None/null values - expected_type = type(DEFAULT_SETTINGS[key]) - actual_type = type(value) - if actual_type != expected_type: - raise StartupError( - f"Setting '{key}' in config file has incorrect type. " - f"Expected {expected_type.__name__}, got {actual_type.__name__}. " - f"Value: {value!r}. " - f"Hint: In YAML/JSON config files, remove quotes from boolean and integer values." - ) - - # Validate settings from constructor parameter - if settings: - for key, value in settings.items(): - if key not in DEFAULT_SETTINGS: - raise StartupError(f"Invalid setting '{key}' in settings parameter") - if value is not None: - expected_type = type(DEFAULT_SETTINGS[key]) - actual_type = type(value) - if actual_type != expected_type: - raise StartupError( - f"Setting '{key}' in settings parameter has incorrect type. " - f"Expected {expected_type.__name__}, got {actual_type.__name__}. " - f"Value: {value!r}" - ) - + raise StartupError("Invalid setting '{}' in datasette.json".format(key)) self.config = config # CLI settings should overwrite datasette.json settings self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) @@ -550,8 +457,6 @@ class Datasette: self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) - self.root_enabled = False - self.default_deny = default_deny self.client = DatasetteClient(self) async def apply_metadata_json(self): @@ -588,9 +493,6 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file - async def _save_queries_from_config(self): - await stored_queries.save_queries_from_config(self) - def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: @@ -600,23 +502,21 @@ class Datasette: pass return environment - def get_action(self, name_or_abbr: str): + def get_permission(self, name_or_abbr: str) -> "Permission": """ - Returns an Action object for the given name or abbreviation. Returns None if not found. + Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. """ - if name_or_abbr in self.actions: - return self.actions[name_or_abbr] + if name_or_abbr in self.permissions: + return self.permissions[name_or_abbr] # Try abbreviation - for action in self.actions.values(): - if action.abbr == name_or_abbr: - return action - return None + for permission in self.permissions.values(): + if permission.abbr == name_or_abbr: + return permission + raise KeyError( + "No permission found with name or abbreviation {}".format(name_or_abbr) + ) async def refresh_schemas(self): - # Throttle schema refreshes to at most once per second - if time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0: - return - self._last_schema_refresh = time.monotonic() if self._refresh_schemas_lock.locked(): return async with self._refresh_schemas_lock: @@ -634,36 +534,6 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } - catalog_table_names = ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - "catalog_databases", - ) - # Delete stale entries for databases that are no longer attached - catalog_database_names = set(current_schema_versions.keys()) - for table in catalog_table_names[:-1]: - catalog_database_names.update( - row["database_name"] - for row in await internal_db.execute( - "select distinct database_name from {}".format(table) - ) - if row["database_name"] is not None - ) - stale_databases = catalog_database_names - set(self.databases.keys()) - if stale_databases: - - def delete_stale_database_catalog(conn): - for stale_db_name in stale_databases: - for table in catalog_table_names: - conn.execute( - "DELETE FROM {} WHERE database_name = ?".format(table), - [stale_db_name], - ) - - await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it @@ -678,7 +548,9 @@ class Datasette: """ INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) VALUES {} - """.format(placeholders), + """.format( + placeholders + ), values, ) await populate_schema_tables(internal_db, db) @@ -687,17 +559,6 @@ class Datasette: def urls(self): return Urls(self) - @property - def pm(self): - """ - Return the global plugin manager instance. - - This provides access to the pluggy PluginManager that manages all - Datasette plugins and hooks. Use datasette.pm.hook.hook_name() to - call plugin hooks. - """ - return pm - async def invoke_startup(self): # This must be called for Datasette to be in a usable state if self._startup_invoked: @@ -710,50 +571,28 @@ class Datasette: event_classes.extend(extra_classes) self.event_classes = tuple(event_classes) - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): + # Register permissions, but watch out for duplicate name/abbr + names = {} + abbrs = {} + for hook in pm.hook.register_permissions(datasette=self): if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): + for p in hook: + if p.name in names and p != names[p.name]: raise StartupError( - "Duplicate action name: {}".format(action.name) + "Duplicate permission name: {}".format(p.name) ) - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): + if p.abbr and p.abbr in abbrs and p != abbrs[p.abbr]: raise StartupError( - "Duplicate action abbr: {}".format(action.abbr) + "Duplicate permission abbr: {}".format(p.abbr) ) - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action - - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError(f"Duplicate column type name: {ct_cls.name}") - self._column_types[ct_cls.name] = ct_cls - + names[p.name] = p + if p.abbr: + abbrs[p.abbr] = p + self.permissions[p.name] = p for hook in pm.hook.prepare_jinja2_environment( env=self._jinja_env, datasette=self ): await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) self._startup_invoked = True @@ -764,78 +603,44 @@ class Datasette: def unsign(self, signed, namespace="default"): return URLSafeSerializer(self._secret, namespace).loads(signed) - def in_client(self) -> bool: - """Check if the current code is executing within a datasette.client request. - - Returns: - bool: True if currently executing within a datasette.client request, False otherwise. - """ - return _in_datasette_client.get() - - def _token_handlers(self): - """Collect all registered token handlers from plugins.""" - from datasette.tokens import TokenHandler - - handlers = [] - for result in pm.hook.register_token_handler(datasette=self): - if isinstance(result, TokenHandler): - handlers.append(result) - elif isinstance(result, list): - handlers.extend(h for h in result if isinstance(h, TokenHandler)) - return handlers - - async def create_token( + def create_token( self, actor_id: str, *, - expires_after: int | None = None, - restrictions: "TokenRestrictions | None" = None, - handler: str | None = None, - ) -> str: - """ - Create an API token for the given actor. + expires_after: Optional[int] = None, + restrict_all: Optional[Iterable[str]] = None, + restrict_database: Optional[Dict[str, Iterable[str]]] = None, + restrict_resource: Optional[Dict[str, Dict[str, Iterable[str]]]] = None, + ): + token = {"a": actor_id, "t": int(time.time())} + if expires_after: + token["d"] = expires_after - Uses the first registered token handler by default, or a specific - handler if ``handler`` is provided (matched by handler name). + def abbreviate_action(action): + # rename to abbr if possible + permission = self.permissions.get(action) + if not permission: + return action + return permission.abbr or action - Pass a :class:`TokenRestrictions` to limit which actions the token - can perform. - """ - handlers = self._token_handlers() - if not handlers: - raise RuntimeError("No token handlers are registered") - - if handler is not None: - matched = [h for h in handlers if h.name == handler] - if not matched: - available = [h.name for h in handlers] - raise ValueError( - f"Token handler {handler!r} not found. " - f"Available handlers: {available}" - ) - chosen = matched[0] - else: - chosen = handlers[0] - - return await chosen.create_token( - self, - actor_id, - expires_after=expires_after, - restrictions=restrictions, - ) - - async def verify_token(self, token: str) -> dict | None: - """ - Verify an API token by trying all registered token handlers. - - Returns an actor dict from the first handler that recognizes the - token, or None if no handler accepts it. - """ - for token_handler in self._token_handlers(): - result = await token_handler.verify_token(self, token) - if result is not None: - return result - return None + if expires_after: + token["d"] = expires_after + if restrict_all or restrict_database or restrict_resource: + token["_r"] = {} + if restrict_all: + token["_r"]["a"] = [abbreviate_action(a) for a in restrict_all] + if restrict_database: + token["_r"]["d"] = {} + for database, actions in restrict_database.items(): + token["_r"]["d"][database] = [abbreviate_action(a) for a in actions] + if restrict_resource: + token["_r"]["r"] = {} + for database, resources in restrict_resource.items(): + for resource, actions in resources.items(): + token["_r"]["r"].setdefault(database, {})[resource] = [ + abbreviate_action(a) for a in actions + ] + return "dstok_{}".format(self.sign(token, namespace="token")) def get_database(self, name=None, route=None): if route is not None: @@ -866,10 +671,8 @@ class Datasette: self.databases = new_databases return db - def add_memory_database(self, memory_name, name=None, route=None): - return self.add_database( - Database(self, memory_name=memory_name), name=name, route=route - ) + def add_memory_database(self, memory_name): + return self.add_database(Database(self, memory_name=memory_name)) def remove_database(self, name): self.get_database(name).close() @@ -877,33 +680,6 @@ class Datasette: new_databases.pop(name) self.databases = new_databases - def close(self): - """Release all resources held by this Datasette instance. - - Closes every attached Database (including the internal database), - shuts down the executor, and unlinks the temporary file used for - the internal database if one was created. Idempotent and one-way. - """ - if self._closed: - return - self._closed = True - first_exception = None - dbs = list(self.databases.values()) + [self._internal_database] - for db in dbs: - try: - db.close() - except Exception as e: - if first_exception is None: - first_exception = e - if self.executor is not None: - try: - self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: - if first_exception is None: - first_exception = e - if first_exception is not None: - raise first_exception - def setting(self, key): return self._settings.get(key, None) @@ -923,12 +699,14 @@ class Datasette: return orig async def get_instance_metadata(self): - rows = await self.get_internal_database().execute(""" + rows = await self.get_internal_database().execute( + """ SELECT key, value FROM metadata_instance - """) + """ + ) return dict(rows) async def get_database_metadata(self, database_name: str): @@ -1028,349 +806,6 @@ class Datasette: [database_name, resource_name, column_name, key, value], ) - @staticmethod - def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: - return stored_queries.query_row_to_stored_query(row) - - @staticmethod - def _query_options_json(options): - return stored_queries.query_options_json(options) - - async def add_query( - self, - database: str, - name: str, - sql: str, - *, - title: str | None = None, - description: str | None = None, - description_html: str | None = None, - hide_sql: bool = False, - fragment: str | None = None, - parameters: Iterable[str] | None = None, - is_write: bool = False, - is_private: bool = False, - is_trusted: bool = False, - source: str = "plugin", - owner_id: str | None = None, - on_success_message: str | None = None, - on_success_message_sql: str | None = None, - on_success_redirect: str | None = None, - on_error_message: str | None = None, - on_error_redirect: str | None = None, - replace: bool = True, - ) -> None: - return await stored_queries.add_query( - self, - database, - name, - sql, - title=title, - description=description, - description_html=description_html, - hide_sql=hide_sql, - fragment=fragment, - parameters=parameters, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - on_success_message=on_success_message, - on_success_message_sql=on_success_message_sql, - on_success_redirect=on_success_redirect, - on_error_message=on_error_message, - on_error_redirect=on_error_redirect, - replace=replace, - ) - - async def update_query( - self, - database: str, - name: str, - *, - sql=stored_queries.UNCHANGED, - title=stored_queries.UNCHANGED, - description=stored_queries.UNCHANGED, - description_html=stored_queries.UNCHANGED, - hide_sql=stored_queries.UNCHANGED, - fragment=stored_queries.UNCHANGED, - parameters=stored_queries.UNCHANGED, - is_write=stored_queries.UNCHANGED, - is_private=stored_queries.UNCHANGED, - is_trusted=stored_queries.UNCHANGED, - source=stored_queries.UNCHANGED, - owner_id=stored_queries.UNCHANGED, - on_success_message=stored_queries.UNCHANGED, - on_success_message_sql=stored_queries.UNCHANGED, - on_success_redirect=stored_queries.UNCHANGED, - on_error_message=stored_queries.UNCHANGED, - on_error_redirect=stored_queries.UNCHANGED, - ) -> None: - return await stored_queries.update_query( - self, - database, - name, - sql=sql, - title=title, - description=description, - description_html=description_html, - hide_sql=hide_sql, - fragment=fragment, - parameters=parameters, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - on_success_message=on_success_message, - on_success_message_sql=on_success_message_sql, - on_success_redirect=on_success_redirect, - on_error_message=on_error_message, - on_error_redirect=on_error_redirect, - ) - - async def remove_query( - self, database: str, name: str, source: str | None = None - ) -> None: - return await stored_queries.remove_query(self, database, name, source=source) - - async def get_query( - self, database: str, name: str - ) -> stored_queries.StoredQuery | None: - return await stored_queries.get_query(self, database, name) - - async def count_queries( - self, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - ) -> int: - return await stored_queries.count_queries( - self, - database, - actor=actor, - q=q, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - ) - - async def list_queries( - self, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - limit: int = 50, - cursor: str | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - include_private: bool = False, - ) -> stored_queries.StoredQueryPage: - return await stored_queries.list_queries( - self, - database, - actor=actor, - limit=limit, - cursor=cursor, - q=q, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - include_private=include_private, - ) - - async def ensure_query_write_permissions( - self, database, sql, *, actor=None, params=None, analysis=None - ): - # Raise Forbidden or QueryWriteRejected if SQL should not run - return await write_sql.ensure_query_write_permissions( - self, database, sql, actor=actor, params=params, analysis=analysis - ) - - # Column types API - - async def _get_resource_column_details(self, database: str, resource: str): - db = self.databases.get(database) - if db is None: - return {} - try: - return { - column.name: column - for column in await db.table_column_details(resource) - } - except sqlite3.OperationalError: - return {} - - @staticmethod - def _column_type_is_applicable(ct_cls, column_detail) -> bool: - sqlite_types = getattr(ct_cls, "sqlite_types", None) - if sqlite_types is None: - return True - if column_detail is None: - return False - actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type) - return actual_sqlite_type in sqlite_types - - async def _validate_column_type_assignment( - self, database: str, resource: str, column: str, ct_cls - ) -> None: - sqlite_types = getattr(ct_cls, "sqlite_types", None) - if sqlite_types is None: - return - - column_detail = ( - await self._get_resource_column_details(database, resource) - ).get(column) - if column_detail is None: - return - - actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type) - if actual_sqlite_type in sqlite_types: - return - - allowed = ", ".join(sqlite_type.value for sqlite_type in sqlite_types) - actual = ( - actual_sqlite_type.value - if actual_sqlite_type is not None - else "unrecognized {!r}".format(column_detail.type) - ) - raise ValueError( - "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " - "has SQLite type {}".format( - ct_cls.name, - allowed, - database, - resource, - column, - actual, - ) - ) - - async def _apply_column_types_config(self): - """Load column_types from datasette.json config into the internal DB.""" - import logging - - for db_name, db_conf in (self.config or {}).get("databases", {}).items(): - for table_name, table_conf in db_conf.get("tables", {}).items(): - for col_name, ct in table_conf.get("column_types", {}).items(): - if isinstance(ct, str): - col_type, config = ct, None - else: - col_type = ct["type"] - config = ct.get("config") - if col_type not in self._column_types: - logging.warning( - "column_types config references unknown type %r " - "for %s.%s.%s", - col_type, - db_name, - table_name, - col_name, - ) - try: - await self.set_column_type( - db_name, table_name, col_name, col_type, config - ) - except ValueError as ex: - logging.warning(str(ex)) - - async def get_column_type(self, database: str, resource: str, column: str): - """ - Return a ColumnType instance (with config baked in) for a specific - column, or None if no column type is assigned. - """ - row = await self.get_internal_database().execute( - "SELECT column_type, config FROM column_types " - "WHERE database_name = ? AND resource_name = ? AND column_name = ?", - [database, resource, column], - ) - rows = row.rows - if not rows: - return None - ct_name, config = rows[0] - ct_cls = self._column_types.get(ct_name) - if ct_cls is None: - return None - column_detail = ( - await self._get_resource_column_details(database, resource) - ).get(column) - if not self._column_type_is_applicable(ct_cls, column_detail): - return None - return ct_cls(config=json.loads(config) if config else None) - - async def get_column_types(self, database: str, resource: str) -> dict: - """ - Return {column_name: ColumnType instance (with config)} - for all columns with assigned types on the given resource. - """ - rows = await self.get_internal_database().execute( - "SELECT column_name, column_type, config FROM column_types " - "WHERE database_name = ? AND resource_name = ?", - [database, resource], - ) - column_details = await self._get_resource_column_details(database, resource) - result = {} - for row in rows.rows: - col_name, ct_name, config = row - ct_cls = self._column_types.get(ct_name) - if ct_cls is not None and self._column_type_is_applicable( - ct_cls, column_details.get(col_name) - ): - result[col_name] = ct_cls(config=json.loads(config) if config else None) - return result - - async def set_column_type( - self, - database: str, - resource: str, - column: str, - column_type: str, - config: dict = None, - ) -> None: - """Assign a column type. Overwrites any existing assignment.""" - ct_cls = self._column_types.get(column_type) - if ct_cls is not None: - await self._validate_column_type_assignment( - database, resource, column, ct_cls - ) - await self.get_internal_database().execute_write( - """INSERT OR REPLACE INTO column_types - (database_name, resource_name, column_name, column_type, config) - VALUES (?, ?, ?, ?, ?)""", - [ - database, - resource, - column, - column_type, - json.dumps(config) if config else None, - ], - ) - - async def remove_column_type( - self, database: str, resource: str, column: str - ) -> None: - """Remove a column type assignment.""" - await self.get_internal_database().execute_write( - "DELETE FROM column_types " - "WHERE database_name = ? AND resource_name = ? AND column_name = ?", - [database, resource, column], - ) - def get_internal_database(self): return self._internal_database @@ -1414,24 +849,38 @@ class Datasette: return db_plugin_config - def static_hash(self, filename): - if not hasattr(self, "_static_hashes"): - self._static_hashes = {} - path = os.path.join(str(app_root), "datasette/static", filename) - signature = (os.path.getmtime(path), os.path.getsize(path)) - cached = self._static_hashes.get(filename) - if cached and cached["signature"] == signature: - return cached["hash"] - with open(path) as fp: - static_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[:6] - self._static_hashes[filename] = { - "signature": signature, - "hash": static_hash, - } - return static_hash - def app_css_hash(self): - return self.static_hash("app.css") + if not hasattr(self, "_app_css_hash"): + with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: + self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ + :6 + ] + return self._app_css_hash + + async def get_canned_queries(self, database_name, actor): + queries = ( + ((self.config or {}).get("databases") or {}).get(database_name) or {} + ).get("queries") or {} + for more_queries in pm.hook.canned_queries( + datasette=self, + database=database_name, + actor=actor, + ): + more_queries = await await_me_maybe(more_queries) + queries.update(more_queries or {}) + # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}} + for key in queries: + if not isinstance(queries[key], dict): + queries[key] = {"sql": queries[key]} + # Also make sure "name" is available: + queries[key]["name"] = key + return queries + + async def get_canned_query(self, database_name, query_name, actor): + queries = await self.get_canned_queries(database_name, actor) + query = queries.get(query_name) + if query: + return query def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row @@ -1493,14 +942,14 @@ class Datasette: if request: actor = request.actor # Top-level link - if await self.allowed(action="view-instance", actor=actor): + if await self.permission_allowed(actor=actor, action="view-instance"): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link if database: - if await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), + if await self.permission_allowed( actor=actor, + action="view-database", + resource=database, ): crumbs.append( { @@ -1511,10 +960,10 @@ class Datasette: # Table link if table: assert database, "table= requires database=" - if await self.allowed( - action="view-table", - resource=TableResource(database=database, table=table), + if await self.permission_allowed( actor=actor, + action="view-table", + resource=(database, table), ): crumbs.append( { @@ -1525,8 +974,8 @@ class Datasette: return crumbs async def actors_from_ids( - self, actor_ids: Iterable[str | int] - ) -> Dict[int | str, Dict]: + self, actor_ids: Iterable[Union[str, int]] + ) -> Dict[Union[id, str], Dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1541,354 +990,257 @@ class Datasette: for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) - def resource_for_action(self, action: str, parent: str | None, child: str | None): - """ - Create a Resource instance for the given action with parent/child values. - - Looks up the action's resource_class and instantiates it with the - provided parent and child identifiers. - - Args: - action: The action name (e.g., "view-table", "view-query") - parent: The parent resource identifier (e.g., database name) - child: The child resource identifier (e.g., table/query name) - - Returns: - A Resource instance of the appropriate subclass - - Raises: - ValueError: If the action is unknown - """ - from datasette.permissions import Resource - - action_obj = self.actions.get(action) - if not action_obj: - raise ValueError(f"Unknown action: {action}") - - resource_class = action_obj.resource_class - instance = object.__new__(resource_class) - Resource.__init__(instance, parent=parent, child=child) - return instance - - async def check_visibility( - self, - actor: dict, - action: str, - resource: "Resource" | None = None, + async def permission_allowed( + self, actor, action, resource=None, *, default=DEFAULT_NOT_SET ): - """ - Check if actor can see a resource and if it's private. - - Returns (visible, private) tuple: - - visible: bool - can the actor see it? - - private: bool - if visible, can anonymous users NOT see it? - """ - from datasette.permissions import Resource - - # Validate that resource is a Resource object or None - if resource is not None and not isinstance(resource, Resource): - raise TypeError("resource must be a Resource subclass instance or None.") - - # Check if actor can see it - if not await self.allowed(action=action, resource=resource, actor=actor): - return False, False - - # Check if anonymous user can see it (for "private" flag) - if not await self.allowed(action=action, resource=resource, actor=None): - # Actor can see it but anonymous cannot - it's private - return True, True - - # Both actor and anonymous can see it - it's public - return True, False - - async def allowed_resources_sql( - self, - *, - action: str, - actor: dict | None = None, - parent: str | None = None, - include_is_private: bool = False, - ) -> ResourcesSQL: - """ - Build SQL query to get all resources the actor can access for the given action. - - Args: - action: The action name (e.g., "view-table") - actor: The actor dict (or None for unauthenticated) - parent: Optional parent filter (e.g., database name) to limit results - include_is_private: If True, include is_private column showing if anonymous cannot access - - Returns a namedtuple of (query: str, params: dict) that can be executed against the internal database. - The query returns rows with (parent, child, reason) columns, plus is_private if requested. - - Example: - query, params = await datasette.allowed_resources_sql( - action="view-table", - actor=actor, - parent="mydb", - include_is_private=True - ) - result = await datasette.get_internal_database().execute(query, params) - """ - from datasette.utils.actions_sql import build_allowed_resources_sql - - action_obj = self.actions.get(action) - if not action_obj: - raise ValueError(f"Unknown action: {action}") - - sql, params = await build_allowed_resources_sql( - self, actor, action, parent=parent, include_is_private=include_is_private - ) - return ResourcesSQL(sql, params) - - async def allowed_resources( - self, - action: str, - actor: dict | None = None, - *, - parent: str | None = None, - include_is_private: bool = False, - include_reasons: bool = False, - limit: int = 100, - next: str | None = None, - ) -> PaginatedResources: - """ - Return paginated resources the actor can access for the given action. - - Uses SQL with keyset pagination to efficiently filter resources. - Returns PaginatedResources with list of Resource instances and pagination metadata. - - Args: - action: The action name (e.g., "view-table") - actor: The actor dict (or None for unauthenticated) - parent: Optional parent filter (e.g., database name) to limit results - include_is_private: If True, adds a .private attribute to each Resource - include_reasons: If True, adds a .reasons attribute with List[str] of permission reasons - limit: Maximum number of results to return (1-1000, default 100) - next: Keyset token from previous page for pagination - - Returns: - PaginatedResources with: - - resources: List of Resource objects for this page - - next: Token for next page (None if no more results) - - Example: - # Get first page of tables - page = await datasette.allowed_resources("view-table", actor, limit=50) - for table in page.resources: - print(f"{table.parent}/{table.child}") - - # Get next page - if page.next: - next_page = await datasette.allowed_resources( - "view-table", actor, limit=50, next=page.next - ) - - # With reasons for debugging - page = await datasette.allowed_resources( - "view-table", actor, include_reasons=True - ) - for table in page.resources: - print(f"{table.child}: {table.reasons}") - - # Iterate through all results with async generator - page = await datasette.allowed_resources("view-table", actor) - async for table in page.all(): - print(table.child) - """ - - action_obj = self.actions.get(action) - if not action_obj: - raise ValueError(f"Unknown action: {action}") - - # Validate and cap limit - limit = min(max(1, limit), 1000) - - # Get base SQL query - query, params = await self.allowed_resources_sql( - action=action, - actor=actor, - parent=parent, - include_is_private=include_is_private, - ) - - # Add keyset pagination WHERE clause if next token provided - if next: - try: - components = urlsafe_components(next) - if len(components) >= 2: - last_parent, last_child = components[0], components[1] - # Keyset condition: (parent > last) OR (parent = last AND child > last) - keyset_where = """ - (parent > :keyset_parent OR - (parent = :keyset_parent AND child > :keyset_child)) - """ - # Wrap original query and add keyset filter - query = f"SELECT * FROM ({query}) WHERE {keyset_where}" - params["keyset_parent"] = last_parent - params["keyset_child"] = last_child - except (ValueError, KeyError): - # Invalid token - ignore and start from beginning - pass - - # Add LIMIT (fetch limit+1 to detect if there are more results) - # Note: query from allowed_resources_sql() already includes ORDER BY parent, child - query = f"{query} LIMIT :limit" - params["limit"] = limit + 1 - - # Execute query - result = await self.get_internal_database().execute(query, params) - rows = list(result.rows) - - # Check if truncated (got more than limit rows) - truncated = len(rows) > limit - if truncated: - rows = rows[:limit] # Remove the extra row - - # Build Resource objects with optional attributes - resources = [] - for row in rows: - # row[0]=parent, row[1]=child, row[2]=reason, row[3]=is_private (if requested) - resource = self.resource_for_action(action, parent=row[0], child=row[1]) - - # Add reasons if requested - if include_reasons: - reason_json = row[2] - try: - reasons_array = ( - json.loads(reason_json) if isinstance(reason_json, str) else [] - ) - resource.reasons = [r for r in reasons_array if r is not None] - except (json.JSONDecodeError, TypeError): - resource.reasons = [reason_json] if reason_json else [] - - # Add private flag if requested - if include_is_private: - resource.private = bool(row[3]) - - resources.append(resource) - - # Generate next token if there are more results - next_token = None - if truncated and resources: - last_resource = resources[-1] - # Use tilde-encoding like table pagination - next_token = "{},{}".format( - tilde_encode(str(last_resource.parent)), - tilde_encode(str(last_resource.child)), - ) - - return PaginatedResources( - resources=resources, - next=next_token, - _datasette=self, - _action=action, - _actor=actor, - _parent=parent, - _include_is_private=include_is_private, - _include_reasons=include_reasons, - _limit=limit, - ) - - async def allowed( - self, - *, - action: str, - resource: "Resource" = None, - actor: dict | None = None, - ) -> bool: - """ - Check if actor can perform action on specific resource. - - Uses SQL to check permission for a single resource without fetching all resources. - This is efficient - it does NOT call allowed_resources() and check membership. - - For global actions, resource should be None (or omitted). - - Example: - from datasette.resources import TableResource - can_view = await datasette.allowed( - action="view-table", - resource=TableResource(database="analytics", table="users"), - actor=actor - ) - - # For global actions, resource can be omitted: - can_debug = await datasette.allowed(action="permissions-debug", actor=actor) - """ - from datasette.utils.actions_sql import check_permission_for_resource - - # For global actions, resource remains None - - # Check if this action has also_requires - if so, check that action first - action_obj = self.actions.get(action) - if action_obj and action_obj.also_requires: - # Must have the required action first - if not await self.allowed( - action=action_obj.also_requires, - resource=resource, - actor=actor, - ): - return False - - # For global actions, resource is None - parent = resource.parent if resource else None - child = resource.child if resource else None - - result = await check_permission_for_resource( + """Check permissions using the permissions_allowed plugin hook""" + result = None + # Use default from registered permission, if available + if default is DEFAULT_NOT_SET and action in self.permissions: + default = self.permissions[action].default + opinions = [] + # Every plugin is consulted for their opinion + for check in pm.hook.permission_allowed( datasette=self, actor=actor, action=action, - parent=parent, - child=child, - ) + resource=resource, + ): + check = await await_me_maybe(check) + if check is not None: + opinions.append(check) - # Log the permission check for debugging + result = None + # If any plugin said False it's false - the veto rule + if any(not r for r in opinions): + result = False + elif any(r for r in opinions): + # Otherwise, if any plugin said True it's true + result = True + + used_default = False + if result is None: + # No plugin expressed an opinion, so use the default + result = default + used_default = True self._permission_checks.append( - PermissionCheck( - when=datetime.datetime.now(datetime.timezone.utc).isoformat(), - actor=actor, - action=action, - parent=parent, - child=child, - result=result, - ) + { + "when": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "actor": actor, + "action": action, + "resource": resource, + "used_default": used_default, + "result": result, + } + ) + return result + + async def allowed_resources_sql( + self, actor: dict | None, action: str + ) -> tuple[str, dict]: + """Combine permission_resources_sql PluginSQL blocks into a UNION query. + + Returns a (sql, params) tuple suitable for execution against SQLite. + """ + plugin_blocks: List[PluginSQL] = [] + for block in pm.hook.permission_resources_sql( + datasette=self, + actor=actor, + action=action, + ): + block = await await_me_maybe(block) + if block is None: + continue + if isinstance(block, (list, tuple)): + candidates = block + else: + candidates = [block] + for candidate in candidates: + if candidate is None: + continue + if not isinstance(candidate, PluginSQL): + continue + plugin_blocks.append(candidate) + + actor_id = actor.get("id") if actor else None + sql, params = build_rules_union( + actor=str(actor_id) if actor_id is not None else "", + plugins=plugin_blocks, + ) + return sql, params + + async def permission_allowed_2( + self, actor, action, resource=None, *, default=DEFAULT_NOT_SET + ): + """Permission check backed by permission_resources_sql rules.""" + + if default is DEFAULT_NOT_SET and action in self.permissions: + default = self.permissions[action].default + + if isinstance(actor, dict) or actor is None: + actor_dict = actor + else: + actor_dict = {"id": actor} + actor_id = actor_dict.get("id") if actor_dict else None + + candidate_parent = None + candidate_child = None + if isinstance(resource, str): + candidate_parent = resource + elif isinstance(resource, (tuple, list)) and len(resource) == 2: + candidate_parent, candidate_child = resource + elif resource is not None: + raise TypeError("resource must be None, str, or (parent, child) tuple") + + union_sql, union_params = await self.allowed_resources_sql(actor_dict, action) + + query = f""" + WITH rules AS ( + {union_sql} + ), + candidate AS ( + SELECT :cand_parent AS parent, :cand_child AS child + ), + matched AS ( + SELECT + r.allow, + r.reason, + r.source_plugin, + CASE + WHEN r.child IS NOT NULL THEN 2 + WHEN r.parent IS NOT NULL THEN 1 + ELSE 0 + END AS depth + FROM rules r + JOIN candidate c + ON (r.parent IS NULL OR r.parent = c.parent) + AND (r.child IS NULL OR r.child = c.child) + ), + ranked AS ( + SELECT *, + ROW_NUMBER() OVER ( + ORDER BY + depth DESC, + CASE WHEN allow = 0 THEN 0 ELSE 1 END, + source_plugin + ) AS rn + FROM matched + ), + winner AS ( + SELECT allow, reason, source_plugin, depth + FROM ranked + WHERE rn = 1 + ) + SELECT allow, reason, source_plugin, depth FROM winner + """ + + params = { + **union_params, + "cand_parent": candidate_parent, + "cand_child": candidate_child, + } + + rows = await self.get_internal_database().execute(query, params) + row = rows.first() + + reason = None + source_plugin = None + depth = None + used_default = False + + if row is None: + result = default + used_default = True + else: + allow = row["allow"] + reason = row["reason"] + source_plugin = row["source_plugin"] + depth = row["depth"] + if allow is None: + result = default + used_default = True + else: + result = bool(allow) + + self._permission_checks.append( + { + "when": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "actor": actor, + "action": action, + "resource": resource, + "used_default": used_default, + "result": result, + "reason": reason, + "source_plugin": source_plugin, + "depth": depth, + } ) return result - async def ensure_permission( + async def ensure_permissions( self, - *, - action: str, - resource: "Resource" = None, - actor: dict | None = None, + actor: dict, + permissions: Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]], ): """ - Check if actor can perform action on resource, raising Forbidden if not. + permissions is a list of (action, resource) tuples or 'action' strings - This is a convenience wrapper around allowed() that raises Forbidden - instead of returning False. Use this when you want to enforce a permission - check and halt execution if it fails. - - Example: - from datasette.resources import TableResource - - # Will raise Forbidden if actor cannot view the table - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database="analytics", table="users"), - actor=request.actor - ) - - # For instance-level actions, resource can be omitted: - await datasette.ensure_permission( - action="permissions-debug", - actor=request.actor - ) + Raises datasette.Forbidden() if any of the checks fail """ - if not await self.allowed(action=action, resource=resource, actor=actor): - raise Forbidden(action) + assert actor is None or isinstance(actor, dict), "actor must be None or a dict" + for permission in permissions: + if isinstance(permission, str): + action = permission + resource = None + elif isinstance(permission, (tuple, list)) and len(permission) == 2: + action, resource = permission + else: + assert ( + False + ), "permission should be string or tuple of two items: {}".format( + repr(permission) + ) + ok = await self.permission_allowed( + actor, + action, + resource=resource, + default=None, + ) + if ok is not None: + if ok: + return + else: + raise Forbidden(action) + + async def check_visibility( + self, + actor: dict, + action: Optional[str] = None, + resource: Optional[Union[str, Tuple[str, str]]] = None, + permissions: Optional[ + Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]] + ] = None, + ): + """Returns (visible, private) - visible = can you see it, private = can others see it too""" + if permissions: + assert ( + not action and not resource + ), "Can't use action= or resource= with permissions=" + else: + permissions = [(action, resource)] + try: + await self.ensure_permissions(actor, permissions) + except Forbidden: + return False, False + # User can see it, but can the anonymous user see it? + try: + await self.ensure_permissions(None, permissions) + except Forbidden: + # It's visible but private + return True, True + # It's visible to everyone + return True, False async def execute( self, @@ -1924,14 +1276,15 @@ class Datasette: except IndexError: return {} # Ensure user has permission to view the referenced table - from datasette.resources import TableResource - other_table = fk["other_table"] other_column = fk["other_column"] visible, _ = await self.check_visibility( actor, - action="view-table", - resource=TableResource(database=database, table=other_table), + permissions=[ + ("view-table", (database, other_table)), + ("view-database", database), + "view-instance", + ], ) if not visible: return {} @@ -2051,7 +1404,6 @@ class Datasette: break except importlib.metadata.PackageNotFoundError: pass - conn.close() return info def _plugins(self, request=None, all=False): @@ -2097,22 +1449,6 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - def _actions(self): - return [ - { - "name": action.name, - "abbr": action.abbr, - "description": action.description, - "takes_parent": action.takes_parent, - "takes_child": action.takes_child, - "resource_class": ( - action.resource_class.__name__ if action.resource_class else None - ), - "also_requires": action.also_requires, - } - for action in sorted(self.actions.values(), key=lambda a: a.name) - ] - async def table_config(self, database: str, table: str) -> dict: """Return dictionary of configuration for specified table""" return ( @@ -2146,10 +1482,10 @@ class Datasette: async def render_template( self, - templates: List[str] | str | Template, - context: Dict[str, Any] | Context | None = None, - request: Request | None = None, - view_name: str | None = None, + templates: Union[List[str], str, Template], + context: Optional[Union[Dict[str, Any], Context]] = None, + request: Optional[Request] = None, + view_name: Optional[str] = None, ): if not self._startup_invoked: raise Exception("render_template() called before await ds.invoke_startup()") @@ -2235,11 +1571,7 @@ class Datasette: "extra_js_urls", template, context, request, view_name ), "base_url": self.setting("base_url"), - "csrftoken": ( - request.scope["csrftoken"] - if request and "csrftoken" in request.scope - else lambda: "" - ), + "csrftoken": request.scope["csrftoken"] if request else lambda: "", "datasette_version": __version__, }, **extra_template_vars, @@ -2252,7 +1584,7 @@ class Datasette: return await template.render_async(template_context) def set_actor_cookie( - self, response: Response, actor: dict, expire_after: int | None = None + self, response: Response, actor: dict, expire_after: Optional[int] = None ): data = {"a": actor} if expire_after: @@ -2382,16 +1714,6 @@ class Datasette: ), r"/-/actor(\.(?Pjson))?$", ) - add_route( - JsonDataView.as_view( - self, - "actions.json", - self._actions, - template="debug_actions.html", - permission="permissions-debug", - ), - r"/-/actions(\.(?Pjson))?$", - ) add_route( AuthTokenView.as_view(self), r"/-/auth-token$", @@ -2404,18 +1726,6 @@ class Datasette: ApiExplorerView.as_view(self), r"/-/api$", ) - add_route( - JumpView.as_view(self), - r"/-/jump(\.(?Pjson))?$", - ) - add_route( - GlobalQueryListView.as_view(self), - r"/-/queries(\.(?Pjson))?$", - ) - add_route( - InstanceSchemaView.as_view(self), - r"/-/schema(\.(?Pjson|md))?$", - ) add_route( LogoutView.as_view(self), r"/-/logout$", @@ -2457,50 +1767,10 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") - add_route( - QueryListView.as_view(self), - r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", - ) - add_route( - QueryCreateAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/analyze$", - ) - add_route( - QueryStoreView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/store$", - ) - add_route( - ExecuteWriteAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write/analyze$", - ) - add_route( - ExecuteWriteView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write$", - ) - add_route( - DatabaseSchemaView.as_view(self), - r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", - ) - add_route( - QueryParametersView.as_view(self), - r"/(?P[^\/\.]+)/-/query/parameters$", - ) add_route( wrap_view(QueryView, self), r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", ) - add_route( - QueryDefinitionView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", - ) - add_route( - QueryUpdateView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", - ) - add_route( - QueryDeleteView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/delete$", - ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", @@ -2517,18 +1787,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/upsert$", ) - add_route( - TableSetColumnTypeView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", - ) add_route( TableDropView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", ) - add_route( - TableSchemaView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/schema(\.(?Pjson|md))?$", - ) add_route( RowDeleteView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/delete$", @@ -2582,13 +1844,29 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) - async def _close_on_shutdown(): - self.close() + async def custom_csrf_error(scope, send, message_id): + await asgi_send( + send, + content=await self.render_template( + "csrf_error.html", + {"message_id": message_id, "message_name": Errors(message_id).name}, + ), + status=403, + content_type="text/html; charset=utf-8", + ) - asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) + asgi = asgi_csrf.asgi_csrf( + DatasetteRouter(self, routes), + signing_secret=self._secret, + cookie_name="ds_csrftoken", + skip_if_scope=lambda scope: any( + pm.hook.skip_csrf(datasette=self, scope=scope) + ), + send_csrf_failed=custom_csrf_error, + ) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiLifespan(asgi) asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) @@ -2635,13 +1913,10 @@ class DatasetteRouter: # Handle authentication default_actor = scope.get("actor") or None actor = None - results = pm.hook.actor_from_request(datasette=self.ds, request=request) - for result in results: - result = await await_me_maybe(result) - if result and actor is None: - actor = result - # Don't break — we must await all coroutines to avoid - # "coroutine was never awaited" warnings + for actor in pm.hook.actor_from_request(datasette=self.ds, request=request): + actor = await await_me_maybe(actor) + if actor: + break scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) @@ -2906,18 +2181,9 @@ class NotFoundExplicit(NotFound): class DatasetteClient: - """Internal HTTP client for making requests to a Datasette instance. - - Used for testing and for internal operations that need to make HTTP requests - to the Datasette app without going through an actual HTTP server. - """ - def __init__(self, ds): self.ds = ds - - @property - def app(self): - return self.ds.app() + self.app = ds.app() def actor_cookie(self, actor): # Utility method, mainly for tests @@ -2930,102 +2196,40 @@ class DatasetteClient: path = f"http://localhost{path}" return path - def _apply_actor(self, kwargs): - """If ``actor=`` was supplied, convert it into a signed ds_actor cookie.""" - actor = kwargs.pop("actor", None) - if actor is None: - return - cookies = dict(kwargs.get("cookies") or {}) - if "ds_actor" in cookies: - raise TypeError("Cannot pass both actor= and a ds_actor cookie") - cookies["ds_actor"] = self.actor_cookie(actor) - kwargs["cookies"] = cookies + async def _request(self, method, path, **kwargs): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), + ) as client: + return await getattr(client, method)(self._fix(path), **kwargs) - async def _request(self, method, path, skip_permission_checks=False, **kwargs): - from datasette.permissions import SkipPermissions + async def get(self, path, **kwargs): + return await self._request("get", path, **kwargs) - self._apply_actor(kwargs) - with _DatasetteClientContext(): - if skip_permission_checks: - with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await getattr(client, method)(self._fix(path), **kwargs) - else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await getattr(client, method)(self._fix(path), **kwargs) + async def options(self, path, **kwargs): + return await self._request("options", path, **kwargs) - async def get(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "get", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def head(self, path, **kwargs): + return await self._request("head", path, **kwargs) - async def options(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "options", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def post(self, path, **kwargs): + return await self._request("post", path, **kwargs) - async def head(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "head", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def put(self, path, **kwargs): + return await self._request("put", path, **kwargs) - async def post(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "post", path, skip_permission_checks=skip_permission_checks, **kwargs - ) + async def patch(self, path, **kwargs): + return await self._request("patch", path, **kwargs) - async def put(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "put", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def patch(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "patch", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def delete(self, path, skip_permission_checks=False, **kwargs): - return await self._request( - "delete", path, skip_permission_checks=skip_permission_checks, **kwargs - ) - - async def request(self, method, path, skip_permission_checks=False, **kwargs): - """Make an HTTP request with the specified method. - - Args: - method: HTTP method (e.g., "GET", "POST", "PUT") - path: The path to request - skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx - - Returns: - httpx.Response: The response from the request - """ - from datasette.permissions import SkipPermissions + async def delete(self, path, **kwargs): + return await self._request("delete", path, **kwargs) + async def request(self, method, path, **kwargs): avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) - self._apply_actor(kwargs) - with _DatasetteClientContext(): - if skip_permission_checks: - with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.request( - method, self._fix(path, avoid_path_rewrites), **kwargs - ) - else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.request( - method, self._fix(path, avoid_path_rewrites), **kwargs - ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), + ) as client: + return await client.request( + method, self._fix(path, avoid_path_rewrites), **kwargs + ) diff --git a/datasette/cli.py b/datasette/cli.py index 90a33e80..bacabc4c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,7 +21,6 @@ from .app import ( SQLITE_LIMIT_ATTACHED, pm, ) -from .inspect import inspect_tables from .utils import ( LoadExtension, StartupError, @@ -110,11 +109,15 @@ def sqlite_extensions(fn): return fn(*args, **kwargs) except AttributeError as e: if "enable_load_extension" in str(e): - raise click.ClickException(textwrap.dedent(""" + raise click.ClickException( + textwrap.dedent( + """ Your Python installation does not have the ability to load SQLite extensions. More information: https://datasette.io/help/extensions - """).strip()) + """ + ).strip() + ) raise return wrapped @@ -143,6 +146,7 @@ def inspect(files, inspect_file, sqlite_extensions): This can then be passed to "datasette --inspect-file" to speed up count operations against immutable database files. """ + app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) inspect_data = run_sync(lambda: inspect_(files, sqlite_extensions)) if inspect_file == "-": sys.stdout.write(json.dumps(inspect_data, indent=2)) @@ -155,14 +159,14 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) + counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, "size": database.size, "file": database.path, "tables": { - table_name: {"count": table["count"]} - for table_name, table in tables.items() + table_name: {"count": table_count} + for table_name, table_count in counts.items() }, } return data @@ -435,20 +439,10 @@ def uninstall(packages, yes): help="Output URL that sets a cookie authenticating the root user", is_flag=True, ) -@click.option( - "--default-deny", - help="Deny all permissions by default", - is_flag=True, -) @click.option( "--get", help="Run an HTTP GET request against this path, print results and exit", ) -@click.option( - "--headers", - is_flag=True, - help="Include HTTP headers in --get output", -) @click.option( "--token", help="API token to send with --get requests", @@ -516,9 +510,7 @@ def serve( settings, secret, root, - default_deny, get, - headers, token, actor, version_note, @@ -548,7 +540,7 @@ def serve( if reload: import hupper - reloader = hupper.start_reloader("datasette.cli.cli") + reloader = hupper.start_reloader("datasette.cli.serve") if immutable: reloader.watch_files(immutable) if config: @@ -597,28 +589,18 @@ def serve( crossdb=crossdb, nolock=nolock, internal=internal, - default_deny=default_deny, ) - # Separate directories from files - directories = [f for f in files if os.path.isdir(f)] - file_paths = [f for f in files if not os.path.isdir(f)] - - # Handle config_dir - only one directory allowed - if len(directories) > 1: - raise click.ClickException( - "Cannot pass multiple directories. Pass a single directory as config_dir." - ) - elif len(directories) == 1: - kwargs["config_dir"] = pathlib.Path(directories[0]) + # if files is a single directory, use that as config_dir= + if 1 == len(files) and os.path.isdir(files[0]): + kwargs["config_dir"] = pathlib.Path(files[0]) + files = [] # Verify list of files, create if needed (and --create) - for file in file_paths: + for file in files: if not pathlib.Path(file).exists(): if create: - conn = sqlite3.connect(file) - conn.execute("vacuum") - conn.close() + sqlite3.connect(file).execute("vacuum") else: raise click.ClickException( "Invalid value for '[FILES]...': Path '{}' does not exist.".format( @@ -626,32 +608,8 @@ def serve( ) ) - # Check for duplicate files by resolving all paths to their absolute forms - # Collect all database files that will be loaded (explicit files + config_dir files) - all_db_files = [] - - # Add explicit files - for file in file_paths: - all_db_files.append((file, pathlib.Path(file).resolve())) - - # Add config_dir databases if config_dir is set - if "config_dir" in kwargs: - config_dir = kwargs["config_dir"] - for ext in ("db", "sqlite", "sqlite3"): - for db_file in config_dir.glob(f"*.{ext}"): - all_db_files.append((str(db_file), db_file.resolve())) - - # Check for duplicates - seen = {} - for original_path, resolved_path in all_db_files: - if resolved_path in seen: - raise click.ClickException( - f"Duplicate database file: '{original_path}' and '{seen[resolved_path]}' " - f"both refer to {resolved_path}" - ) - seen[resolved_path] = original_path - - files = file_paths + # De-duplicate files so 'datasette db.db db.db' only attaches one /db + files = list(dict.fromkeys(files)) try: ds = Datasette(files, **kwargs) @@ -664,43 +622,25 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run async soundness checks before startup hooks, since invoke_startup - # now populates internal tables which requires querying each database - run_sync(lambda: check_databases(ds)) - # Run the "startup" plugin hooks - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) + run_sync(ds.invoke_startup) - if headers and not get: - raise click.ClickException("--headers can only be used with --get") + # Run async soundness checks - but only if we're not under pytest + run_sync(lambda: check_databases(ds)) if token and not get: raise click.ClickException("--token can only be used with --get") if get: client = TestClient(ds) - request_headers = {} + headers = {} if token: - request_headers["Authorization"] = "Bearer {}".format(token) + headers["Authorization"] = "Bearer {}".format(token) cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) - response = client.get(get, headers=request_headers, cookies=cookies) - - if headers: - # Output HTTP status code, headers, two newlines, then the response body - click.echo(f"HTTP/1.1 {response.status}") - for key, value in response.headers.items(): - click.echo(f"{key}: {value}") - if response.text: - click.echo() - click.echo(response.text) - else: - click.echo(response.text) - + response = client.get(get, headers=headers, cookies=cookies) + click.echo(response.text) exit_code = 0 if response.status == 200 else 1 sys.exit(exit_code) return @@ -708,7 +648,6 @@ def serve( # Start the server url = None if root: - ds.root_enabled = True url = "http://{}:{}{}?token={}".format( host, port, ds.urls.path("-/auth-token"), ds._root_token ) @@ -818,10 +757,7 @@ def create_token( ds = Datasette(secret=secret, plugins_dir=plugins_dir) # Run ds.invoke_startup() in an event loop - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) + run_sync(ds.invoke_startup) # Warn about any unknown actions actions = [] @@ -829,30 +765,28 @@ def create_token( actions.extend([p[1] for p in databases]) actions.extend([p[2] for p in resources]) for action in actions: - if not ds.actions.get(action): + if not ds.permissions.get(action): click.secho( f" Unknown permission: {action} ", fg="red", err=True, ) - from datasette.tokens import TokenRestrictions - - restrictions = TokenRestrictions() - for action in alls: - restrictions.allow_all(action) + restrict_database = {} for database, action in databases: - restrictions.allow_database(database, action) + restrict_database.setdefault(database, []).append(action) + restrict_resource = {} for database, resource, action in resources: - restrictions.allow_resource(database, resource, action) - - token = run_sync( - lambda: ds.create_token( - id, - expires_after=expires_after, - restrictions=restrictions, - handler="signed", + restrict_resource.setdefault(database, {}).setdefault(resource, []).append( + action ) + + token = ds.create_token( + id, + expires_after=expires_after, + restrict_all=alls, + restrict_database=restrict_database, + restrict_resource=restrict_resource, ) click.echo(token) if debug: diff --git a/datasette/column_types.py b/datasette/column_types.py deleted file mode 100644 index 7320e1d6..00000000 --- a/datasette/column_types.py +++ /dev/null @@ -1,83 +0,0 @@ -from enum import Enum - - -class SQLiteType(Enum): - TEXT = "TEXT" - INTEGER = "INTEGER" - REAL = "REAL" - BLOB = "BLOB" - NULL = "NULL" - - @classmethod - def from_declared_type(cls, declared_type: str | None) -> "SQLiteType | None": - if declared_type is None: - return cls.NULL - - normalized = declared_type.strip().upper() - if not normalized: - return cls.NULL - - if normalized == cls.NULL.value: - return cls.NULL - if "INT" in normalized: - return cls.INTEGER - if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")): - return cls.TEXT - if "BLOB" in normalized: - return cls.BLOB - if any( - token in normalized - for token in ("REAL", "FLOA", "DOUB") # codespell:ignore doub - ): - return cls.REAL - - return None - - -class ColumnType: - """ - Base class for column types. - - Subclasses must define ``name`` and ``description`` as class attributes: - - - ``name``: Unique identifier string. Lowercase, no spaces. - Examples: "markdown", "file", "email", "url", "point", "image". - - ``description``: Human-readable label for admin UI dropdowns. - Examples: "Markdown text", "File reference", "Email address". - - ``sqlite_types``: Optional tuple of SQLiteType values restricting - which SQLite column types this ColumnType can be assigned to. - - Instantiate with an optional ``config`` dict to bind per-column - configuration:: - - ct = MyColumnType(config={"key": "value"}) - ct.config # {"key": "value"} - """ - - name: str - description: str - sqlite_types: tuple[SQLiteType, ...] | None = None - - def __init__(self, config=None): - self.config = config - - async def render_cell(self, value, column, table, database, datasette, request): - """ - Return an HTML string to render this cell value, or None to - fall through to the default render_cell plugin hook chain. - """ - return None - - async def validate(self, value, datasette): - """ - Validate a value before it is written. Return None if valid, - or a string error message if invalid. - """ - return None - - async def transform_value(self, value, datasette): - """ - Transform a value before it appears in JSON API output. - Return the transformed value. Default: return unchanged. - """ - return value diff --git a/datasette/csrf.py b/datasette/csrf.py deleted file mode 100644 index df239aee..00000000 --- a/datasette/csrf.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Header-based CSRF (Cross-Origin) protection. - -Datasette uses the Sec-Fetch-Site + Origin header approach described in -Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented -in Go 1.25's http.CrossOriginProtection. This replaces the previous -token-based asgi-csrf mechanism. -""" - -from __future__ import annotations - -import secrets -import urllib.parse - -from .utils.asgi import asgi_send - -SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) - -DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443} - - -def _normalize_headers(raw_headers): - """Lowercase header names; for duplicates, last value wins.""" - result = {} - for name, value in raw_headers: - if isinstance(name, str): - name = name.encode("latin-1") - if isinstance(value, str): - value = value.encode("latin-1") - result[name.lower()] = value - return result - - -def _origin_tuple(value): - """ - Parse an origin-like string into ``(scheme, host, port)`` with default - ports filled in. Raises ``ValueError`` for malformed input. - """ - parsed = urllib.parse.urlsplit(value) - scheme = (parsed.scheme or "").lower() - host = (parsed.hostname or "").lower() - if not scheme or not host: - raise ValueError("missing scheme or host in {!r}".format(value)) - port = parsed.port # may raise ValueError on bad ports - if port is None: - port = DEFAULT_PORTS.get(scheme) - if port is None: - raise ValueError("unknown default port for scheme {!r}".format(scheme)) - return scheme, host, port - - -def _install_legacy_csrftoken(scope): - """ - Populate ``scope["csrftoken"]`` with a callable returning a per-request - random token. Provided for plugin compatibility only - core no longer - uses this value for CSRF enforcement. - """ - - def csrftoken(): - if "_datasette_legacy_csrftoken" not in scope: - scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32) - return scope["_datasette_legacy_csrftoken"] - - scope["csrftoken"] = csrftoken - - -class CrossOriginProtectionMiddleware: - """ - Modern CSRF protection using the Sec-Fetch-Site and Origin headers. - - Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's - http.CrossOriginProtection. See https://words.filippo.io/csrf/ - - Unsafe-method requests are allowed through only if they look same-origin. - Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin - and are passed through unchanged - CSRF is a browser-only attack. - """ - - SAFE_METHODS = SAFE_METHODS - - def __init__(self, app, datasette): - self.app = app - self.datasette = datasette - - async def __call__(self, scope, receive, send): - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - _install_legacy_csrftoken(scope) - - if scope.get("method", "GET") in self.SAFE_METHODS: - await self.app(scope, receive, send) - return - - headers = _normalize_headers(scope.get("headers") or []) - - authorization = headers.get(b"authorization", b"").decode("latin-1") - cookie_header = headers.get(b"cookie") - # Bearer-token requests are not ambient browser credentials, so they - # are not CSRF-vulnerable. Narrowly exempt them from the header check - # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; - # schemes like Basic or Digest can be browser-managed and ambient. - # If the request also carries a Cookie header, ambient cookie auth - # could be in play, so do NOT treat it as exempt. - if authorization and not cookie_header: - parts = authorization.split(None, 1) - if parts and parts[0].lower() == "bearer": - await self.app(scope, receive, send) - return - - origin_bytes = headers.get(b"origin") - sec_fetch_site_bytes = headers.get(b"sec-fetch-site") - host_bytes = headers.get(b"host", b"") - origin = origin_bytes.decode("latin-1") if origin_bytes else None - sec_fetch_site = ( - sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None - ) - host = host_bytes.decode("latin-1") - - # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS) - if sec_fetch_site is not None: - if sec_fetch_site in ("same-origin", "none"): - await self.app(scope, receive, send) - return - await self._forbid( - send, - "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( - sec_fetch_site - ), - ) - return - - # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.) - if origin is None: - await self.app(scope, receive, send) - return - - # Fallback for older browsers: Origin must match the request's own - # scheme + host + port. Compare full origin tuples, not host alone. - request_scheme = self._request_scheme(scope) - try: - origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) - except ValueError: - await self._forbid( - send, - "Malformed Origin {!r} or Host {!r}".format(origin, host), - ) - return - - if origin_tuple == expected_tuple: - await self.app(scope, receive, send) - return - - await self._forbid( - send, - "Origin {!r} does not match Host {!r}".format(origin, host), - ) - - def _request_scheme(self, scope): - if self.datasette is not None: - try: - if self.datasette.setting("force_https_urls"): - return "https" - except Exception: - pass - return scope.get("scheme") or "http" - - async def _forbid(self, send, reason): - await asgi_send( - send, - content=await self.datasette.render_template( - "csrf_error.html", {"reason": reason} - ), - status=403, - content_type="text/html; charset=utf-8", - ) diff --git a/datasette/database.py b/datasette/database.py index 10417670..b74f02bb 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,19 +1,15 @@ import asyncio -import atexit from collections import namedtuple -import inspect -import os from pathlib import Path +import janus import queue import sqlite_utils import sys -import tempfile import threading import uuid from .tracer import trace from .utils import ( - call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, @@ -25,8 +21,7 @@ from .utils import ( table_columns, table_column_details, ) -from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_hidden_table_names +from .utils.sqlite import sqlite_version from .inspect import inspect_hash connections = threading.local() @@ -34,13 +29,6 @@ connections = threading.local() AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) -class DatasetteClosedError(RuntimeError): - """Raised when using a Datasette or Database instance after close().""" - - -_SHUTDOWN = object() - - class Database: # For table counts stop at this many rows: count_limit = 10000 @@ -54,7 +42,6 @@ class Database: is_memory=False, memory_name=None, mode=None, - is_temp_disk=False, ): self.name = None self._thread_local_id = f"x{self._thread_local_id_counter}" @@ -65,44 +52,19 @@ class Database: self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name - self.is_temp_disk = is_temp_disk if memory_name is not None: self.is_memory = True - if is_temp_disk: - fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_") - os.close(fd) - self.path = temp_path - self.is_mutable = True - self.mode = "rwc" - self._wal_enabled = False - atexit.register(self._cleanup_temp_file) - else: - self._wal_enabled = False self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None self._write_queue = None - self._closed = False - self._pending_execute_futures = set() - self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] - if not is_temp_disk: - self.mode = mode - - def _check_not_closed(self): - if self._closed: - raise DatasetteClosedError( - "Database {!r} has been closed".format(self.name) - ) - - def _remove_pending_execute_future(self, future): - with self._pending_execute_futures_lock: - self._pending_execute_futures.discard(future) + self.mode = mode @property def cached_table_counts(self): @@ -123,8 +85,6 @@ class Database: return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): - if self.is_temp_disk: - return "_temp_disk" if self.path: return Path(self.path).stem elif self.memory_name: @@ -163,104 +123,30 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) self._all_file_connections.append(conn) - if self.is_temp_disk and not self._wal_enabled: - conn.execute("PRAGMA journal_mode=WAL") - self._wal_enabled = True return conn def close(self): - """Release all resources held by this database. - - Idempotent. After close() further calls to execute()/execute_fn()/ - execute_write()/execute_write_fn() raise DatasetteClosedError. - """ - if self._closed: - return - with self._pending_execute_futures_lock: - if self._closed: - return - self._closed = True - pending_execute_futures = tuple(self._pending_execute_futures) - # Shut down the write thread, if any, via a sentinel. The thread - # drains any writes already queued before the sentinel and then - # closes its own write connection and returns. - write_thread = self._write_thread - if write_thread is not None and self._write_queue is not None: - self._write_queue.put(_SHUTDOWN) - write_thread.join(timeout=10) - if write_thread.is_alive(): - sys.stderr.write( - "Datasette: write thread for {!r} did not exit within 10s\n".format( - self.name - ) - ) - sys.stderr.flush() - for future in pending_execute_futures: - try: - future.result() - except Exception: - pass - # Close anything still tracked in _all_file_connections + # Close all connections - useful to avoid running out of file handles in tests for connection in self._all_file_connections: - try: - connection.close() - except Exception: - pass - self._all_file_connections = [] - # Drop per-thread cached read connections we can reach - try: - delattr(connections, self._thread_local_id) - except AttributeError: - pass - # Close non-threaded-mode cached connections if still open - if self._read_connection is not None: - try: - self._read_connection.close() - except Exception: - pass - self._read_connection = None - if self._write_connection is not None: - try: - self._write_connection.close() - except Exception: - pass - self._write_connection = None - if self.is_temp_disk: - self._cleanup_temp_file() - - def _cleanup_temp_file(self): - if self.is_temp_disk and self.path: - for suffix in ("", "-wal", "-shm"): - try: - os.unlink(self.path + suffix) - except OSError: - pass - - async def execute_write(self, sql, params=None, block=True, request=None): - self._check_not_closed() + connection.close() + async def execute_write(self, sql, params=None, block=True): def _inner(conn): return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn(_inner, block=block, request=request) + results = await self.execute_write_fn(_inner, block=block) return results - async def execute_write_script(self, sql, block=True, request=None): - self._check_not_closed() - + async def execute_write_script(self, sql, block=True): def _inner(conn): return conn.executescript(sql) with trace("sql", database=self.name, sql=sql.strip(), executescript=True): - results = await self.execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + results = await self.execute_write_fn(_inner, block=block) return results - async def execute_write_many(self, sql, params_seq, block=True, request=None): - self._check_not_closed() - + async def execute_write_many(self, sql, params_seq, block=True): def _inner(conn): count = 0 @@ -275,14 +161,11 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - results, count = await self.execute_write_fn( - _inner, block=block, request=request - ) + results, count = await self.execute_write_fn(_inner, block=block) kwargs["count"] = count return results async def execute_isolated_fn(self, fn): - self._check_not_closed() # Open a new connection just for the duration of this function # blocking the write queue to avoid any writes occurring during it if self.ds.executor is None: @@ -302,21 +185,7 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) - async def analyze_sql(self, sql, params=None) -> SQLAnalysis: - self._check_not_closed() - - return await self.execute_isolated_fn( - lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) - ) - - async def execute_write_fn(self, fn, block=True, transaction=True, request=None): - self._check_not_closed() - pending_events = [] - - def track_event(event): - pending_events.append(event) - - fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event) + async def execute_write_fn(self, fn, block=True, transaction=True): if self.ds.executor is None: # non-threaded mode if self._write_connection is None: @@ -324,67 +193,13 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: - result = fn(self._write_connection) + return fn(self._write_connection) else: - result = fn(self._write_connection) + return fn(self._write_connection) else: - result = await self._send_to_write_thread( + return await self._send_to_write_thread( fn, block=block, transaction=transaction ) - if block: - for event in pending_events: - await self.ds.track_event(event) - else: - # For non-blocking writes, spawn a background task to - # dispatch events after the write thread completes - task_id, reply_future = result - - async def _dispatch_events_after_write(): - try: - await reply_future - except Exception: - # if the write failed, don't emit success events - return - for event in pending_events: - await self.ds.track_event(event) - - asyncio.ensure_future(_dispatch_events_after_write()) - result = task_id - return result - - def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): - from .plugins import pm - - # Wrap fn so it receives track_event if its signature supports it. - # Historically fn was called positionally, so any single-parameter - # name (conn, connection, db, ...) worked. Preserve that by only - # switching to keyword dependency injection when the callback - # explicitly opts in by declaring a `track_event` parameter. - original_fn = fn - - if "track_event" in inspect.signature(original_fn).parameters: - - def fn_with_track_event(conn): - return call_with_supported_arguments( - original_fn, conn=conn, track_event=track_event - ) - - fn = fn_with_track_event - - wrappers = pm.hook.write_wrapper( - datasette=self.ds, - database=self.name, - request=request, - transaction=transaction, - ) - wrappers = [w for w in wrappers if w is not None] - if not wrappers: - return fn - # Build the wrapped fn by nesting context manager generators. - # The first wrapper returned by pluggy is outermost. - for wrapper_factory in reversed(wrappers): - fn = _apply_write_wrapper(fn, wrapper_factory, track_event) - return fn async def _send_to_write_thread( self, fn, block=True, isolated_connection=False, transaction=True @@ -400,15 +215,18 @@ class Database: ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - loop = asyncio.get_running_loop() - reply_future = loop.create_future() + reply_queue = janus.Queue() self._write_queue.put( - WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) + WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) ) if block: - return await reply_future + result = await reply_queue.async_q.get() + if isinstance(result, Exception): + raise result + else: + return result else: - return task_id, reply_future + return task_id def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -422,47 +240,38 @@ class Database: conn_exception = e while True: task = self._write_queue.get() - if task is _SHUTDOWN: - if conn is not None: - try: - conn.close() - except Exception: - pass - return - exception = None - result = None if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass + result = conn_exception else: - try: - if task.transaction: - with conn: + if task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + else: + try: + if task.transaction: + with conn: + result = task.fn(conn) + else: result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): - self._check_not_closed() if self.ds.executor is None: # non-threaded mode if self._read_connection is None: @@ -479,12 +288,9 @@ class Database: setattr(connections, self._thread_local_id, conn) return fn(conn) - with self._pending_execute_futures_lock: - self._check_not_closed() - future = self.ds.executor.submit(in_thread) - self._pending_execute_futures.add(future) - future.add_done_callback(self._remove_pending_execute_future) - return await asyncio.wrap_future(future) + return await asyncio.get_event_loop().run_in_executor( + self.ds.executor, in_thread + ) async def execute( self, @@ -496,7 +302,6 @@ class Database: log_sql_errors=True, ): """Executes sql against db_name in a thread""" - self._check_not_closed() page_size = page_size or self.ds.page_size def sql_operation_in_thread(conn): @@ -544,7 +349,7 @@ class Database: def hash(self): if self.cached_hash is not None: return self.cached_hash - elif self.is_mutable or self.is_memory or self.is_temp_disk: + elif self.is_mutable or self.is_memory: return None elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): self.cached_hash = self.ds.inspect_data[self.name]["hash"] @@ -603,12 +408,7 @@ class Database: # But SQLite prior to 3.16.0 doesn't support pragma functions results = await self.execute("PRAGMA database_list;") # {'seq': 0, 'name': 'main', 'file': ''} - return [ - AttachedDatabase(*row) - for row in results.rows - # Filter out the SQLite internal "temp" database, refs #2557 - if row["seq"] > 0 and row["name"] != "temp" - ] + return [AttachedDatabase(*row) for row in results.rows if row["seq"] > 0] async def table_exists(self, table): results = await self.execute( @@ -624,7 +424,7 @@ class Database: async def table_names(self): results = await self.execute( - "select name from sqlite_master where type='table' order by name" + "select name from sqlite_master where type='table'" ) return [r[0] for r in results.rows] @@ -702,7 +502,98 @@ class Database: t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] - hidden_tables += await self.execute_fn(sqlite_hidden_table_names) + if sqlite_version()[1] >= 37: + hidden_tables += [ + x[0] + for x in await self.execute( + """ + with shadow_tables as ( + select name + from pragma_table_list + where [type] = 'shadow' + order by name + ), + core_tables as ( + select name + from sqlite_master + WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') + OR substr(name, 1, 1) == '_' + ), + combined as ( + select name from shadow_tables + union all + select name from core_tables + ) + select name from combined order by 1 + """ + ) + ] + else: + hidden_tables += [ + x[0] + for x in await self.execute( + """ + WITH base AS ( + SELECT name + FROM sqlite_master + WHERE name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') + OR substr(name, 1, 1) == '_' + ), + fts_suffixes AS ( + SELECT column1 AS suffix + FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config')) + ), + fts5_names AS ( + SELECT name + FROM sqlite_master + WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%' + ), + fts5_shadow_tables AS ( + SELECT + printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name + FROM fts5_names + JOIN fts_suffixes + ), + fts3_suffixes AS ( + SELECT column1 AS suffix + FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize')) + ), + fts3_names AS ( + SELECT name + FROM sqlite_master + WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%' + OR sql LIKE '%VIRTUAL TABLE%USING FTS4%' + ), + fts3_shadow_tables AS ( + SELECT + printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name + FROM fts3_names + JOIN fts3_suffixes + ), + final AS ( + SELECT name FROM base + UNION ALL + SELECT name FROM fts5_shadow_tables + UNION ALL + SELECT name FROM fts3_shadow_tables + ) + SELECT name FROM final ORDER BY 1 + """ + ) + ] + # Also hide any FTS tables that have a content= argument + hidden_tables += [ + x[0] + for x in await self.execute( + """ + SELECT name + FROM sqlite_master + WHERE sql LIKE '%VIRTUAL TABLE%' + AND sql LIKE '%USING FTS%' + AND sql LIKE '%content=%' + """ + ) + ] has_spatialite = await self.execute_fn(detect_spatialite) if has_spatialite: @@ -721,11 +612,16 @@ class Database: "KNN", "KNN2", ] + [ - r[0] for r in (await self.execute(""" + r[0] + for r in ( + await self.execute( + """ select name from sqlite_master where name like "idx_%" and type = "table" - """)).rows + """ + ) + ).rows ] return hidden_tables @@ -767,8 +663,6 @@ class Database: tags.append("mutable") if self.is_memory: tags.append("memory") - if self.is_temp_disk: - tags.append("temp_disk") if self.hash: tags.append(f"hash={self.hash}") if self.size is not None: @@ -779,90 +673,17 @@ class Database: return f"" -def _apply_write_wrapper(fn, wrapper_factory, track_event): - """Apply a single write_wrapper context manager around fn. - - ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally - ``track_event``, and returns a generator that yields exactly once. - Code before the yield runs before ``fn(conn)``, code after the yield - runs after. The result of ``fn(conn)`` is sent into the generator - via ``.send()``, and any exception raised by ``fn(conn)`` is thrown - via ``.throw()``. - """ - - def wrapped(conn): - gen = call_with_supported_arguments( - wrapper_factory, conn=conn, track_event=track_event - ) - # Advance to the yield point (run "before" code) - try: - next(gen) - except StopIteration: - # Generator didn't yield — just run fn unchanged - return fn(conn) - - # Execute the actual write - try: - result = fn(conn) - except Exception: - # Throw exception into generator so it can handle it - try: - gen.throw(*sys.exc_info()) - except StopIteration: - pass - # Re-raise the original exception - raise - else: - # Send the result back through the yield - try: - gen.send(result) - except StopIteration: - pass - return result - - return wrapped - - class WriteTask: - __slots__ = ( - "fn", - "task_id", - "loop", - "reply_future", - "isolated_connection", - "transaction", - ) + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") - def __init__( - self, fn, task_id, loop, reply_future, isolated_connection, transaction - ): + def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): self.fn = fn self.task_id = task_id - self.loop = loop - self.reply_future = reply_future + self.reply_queue = reply_queue self.isolated_connection = isolated_connection self.transaction = transaction -def _deliver_write_result(task, result, exception): - # Called from the write thread. Delivers the result back to the - # awaiting coroutine on its event loop via call_soon_threadsafe. - def _set(): - if task.reply_future.done(): - # Awaiter was cancelled; nothing to do. - return - if exception is not None: - task.reply_future.set_exception(exception) - else: - task.reply_future.set_result(result) - - try: - task.loop.call_soon_threadsafe(_set) - except RuntimeError: - # Event loop has been closed; the awaiter is gone. - pass - - class QueryInterrupted(Exception): def __init__(self, e, sql, params): self.e = e diff --git a/datasette/default_actions.py b/datasette/default_actions.py deleted file mode 100644 index 2f78570b..00000000 --- a/datasette/default_actions.py +++ /dev/null @@ -1,133 +0,0 @@ -from datasette import hookimpl -from datasette.permissions import Action -from datasette.resources import ( - DatabaseResource, - TableResource, - QueryResource, -) - - -@hookimpl -def register_actions(): - """Register the core Datasette actions.""" - return ( - # Global actions (no resource_class) - Action( - name="view-instance", - abbr="vi", - description="View Datasette instance", - ), - Action( - name="permissions-debug", - abbr="pd", - description="Access permission debug tool", - ), - Action( - name="debug-menu", - abbr="dm", - description="View debug menu items", - ), - # Database-level actions (parent-level) - Action( - name="view-database", - abbr="vd", - description="View database", - resource_class=DatabaseResource, - ), - Action( - name="view-database-download", - abbr="vdd", - description="Download database file", - resource_class=DatabaseResource, - also_requires="view-database", - ), - Action( - name="execute-sql", - abbr="es", - description="Execute read-only SQL queries", - resource_class=DatabaseResource, - also_requires="view-database", - ), - Action( - name="execute-write-sql", - abbr="ews", - description="Execute writable SQL queries", - resource_class=DatabaseResource, - also_requires="view-database", - ), - Action( - name="create-table", - abbr="ct", - description="Create tables", - resource_class=DatabaseResource, - ), - Action( - name="store-query", - abbr="sq", - description="Create stored queries", - resource_class=DatabaseResource, - also_requires="execute-sql", - ), - # Table-level actions (child-level) - Action( - name="view-table", - abbr="vt", - description="View table", - resource_class=TableResource, - ), - Action( - name="insert-row", - abbr="ir", - description="Insert rows", - resource_class=TableResource, - ), - Action( - name="delete-row", - abbr="dr", - description="Delete rows", - resource_class=TableResource, - ), - Action( - name="update-row", - abbr="ur", - description="Update rows", - resource_class=TableResource, - ), - Action( - name="alter-table", - abbr="at", - description="Alter tables", - resource_class=TableResource, - ), - Action( - name="set-column-type", - abbr="sct", - description="Set column type", - resource_class=TableResource, - ), - Action( - name="drop-table", - abbr="dt", - description="Drop tables", - resource_class=TableResource, - ), - # Query-level actions (child-level) - Action( - name="view-query", - abbr="vq", - description="View named query results", - resource_class=QueryResource, - ), - Action( - name="update-query", - abbr="uq", - description="Update stored queries", - resource_class=QueryResource, - ), - Action( - name="delete-query", - abbr="dq", - description="Delete stored queries", - resource_class=QueryResource, - ), - ) diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py deleted file mode 100644 index 24493994..00000000 --- a/datasette/default_column_types.py +++ /dev/null @@ -1,81 +0,0 @@ -import json -import re - -import markupsafe - -from datasette import hookimpl -from datasette.column_types import ColumnType, SQLiteType - - -class UrlColumnType(ColumnType): - name = "url" - description = "URL" - sqlite_types = (SQLiteType.TEXT,) - - async def render_cell(self, value, column, table, database, datasette, request): - if not value or not isinstance(value, str): - return None - escaped = markupsafe.escape(value.strip()) - return markupsafe.Markup(f'{escaped}') - - async def validate(self, value, datasette): - if value is None or value == "": - return None - if not isinstance(value, str): - return "URL must be a string" - if not re.match(r"^https?://\S+$", value.strip()): - return "Invalid URL" - return None - - -class EmailColumnType(ColumnType): - name = "email" - description = "Email address" - sqlite_types = (SQLiteType.TEXT,) - - async def render_cell(self, value, column, table, database, datasette, request): - if not value or not isinstance(value, str): - return None - escaped = markupsafe.escape(value.strip()) - return markupsafe.Markup(f'{escaped}') - - async def validate(self, value, datasette): - if value is None or value == "": - return None - if not isinstance(value, str): - return "Email must be a string" - if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", value.strip()): - return "Invalid email address" - return None - - -class JsonColumnType(ColumnType): - name = "json" - description = "JSON data" - sqlite_types = (SQLiteType.TEXT,) - - async def render_cell(self, value, column, table, database, datasette, request): - if value is None: - return None - try: - parsed = json.loads(value) if isinstance(value, str) else value - formatted = json.dumps(parsed, indent=2) - escaped = markupsafe.escape(formatted) - return markupsafe.Markup(f"
{escaped}
") - except (json.JSONDecodeError, TypeError): - return None - - async def validate(self, value, datasette): - if value is None or value == "": - return None - if isinstance(value, str): - try: - json.loads(value) - except json.JSONDecodeError: - return "Invalid JSON" - return None - - -@hookimpl -def register_column_types(datasette): - return [UrlColumnType, EmailColumnType, JsonColumnType] diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py deleted file mode 100644 index e0cb3cdf..00000000 --- a/datasette/default_database_actions.py +++ /dev/null @@ -1,24 +0,0 @@ -from datasette import hookimpl -from datasette.resources import DatabaseResource - - -@hookimpl -def database_actions(datasette, actor, database, request): - async def inner(): - if not datasette.get_database(database).is_mutable: - return [] - if not await datasette.allowed( - action="execute-write-sql", - resource=DatabaseResource(database), - actor=actor, - ): - return [] - return [ - { - "href": datasette.urls.database(database) + "/-/execute-write", - "label": "Execute write SQL", - "description": "Run writable SQL with table permission checks.", - } - ] - - return inner diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py deleted file mode 100644 index 6127b2a6..00000000 --- a/datasette/default_debug_menu.py +++ /dev/null @@ -1,75 +0,0 @@ -from datasette import hookimpl -from datasette.jump import JumpSQL - -DEBUG_MENU_ITEMS = ( - ( - "/-/databases", - "Databases", - "List of databases known to this Datasette instance.", - ), - ( - "/-/plugins", - "Installed plugins", - "Review loaded plugins, their versions and their registered hooks.", - ), - ( - "/-/versions", - "Version info", - "Check the Python, SQLite and dependency versions used by this server.", - ), - ( - "/-/settings", - "Settings", - "Inspect the active Datasette settings and configuration values.", - ), - ( - "/-/permissions", - "Debug permissions", - "Test permission checks for actors, actions and resources.", - ), - ( - "/-/messages", - "Debug messages", - "Try out temporary flash messages shown to users.", - ), - ( - "/-/allow-debug", - "Debug allow rules", - "Explore how allow blocks match actors against permission rules.", - ), - ( - "/-/threads", - "Debug threads", - "Inspect worker threads and database tasks.", - ), - ( - "/-/actor", - "Debug actor", - "View the actor object for the current signed-in user.", - ), - ( - "/-/patterns", - "Pattern portfolio", - "Browse Datasette UI patterns.", - ), -) - - -@hookimpl -def jump_items_sql(datasette, actor, request): - async def inner(): - if not await datasette.allowed(action="debug-menu", actor=actor): - return [] - - return [ - JumpSQL.menu_item( - label=label, - url=datasette.urls.path(path), - description=description, - search_text=f"debug {label} {description}", - item_type="debug", - ) - for path, label, description in DEBUG_MENU_ITEMS - ] - - return inner diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py deleted file mode 100644 index d215e7ec..00000000 --- a/datasette/default_jump_items.py +++ /dev/null @@ -1,82 +0,0 @@ -from datasette import hookimpl -from datasette.jump import JumpSQL - - -@hookimpl -def jump_items_sql(datasette, actor, request): - async def inner(): - database_sql, database_params = await datasette.allowed_resources_sql( - action="view-database", actor=actor - ) - table_sql, table_params = await datasette.allowed_resources_sql( - action="view-table", actor=actor - ) - query_sql, query_params = await datasette.allowed_resources_sql( - action="view-query", actor=actor - ) - return [ - JumpSQL( - sql=f""" - WITH allowed_databases AS ( - {database_sql} - ) - SELECT - 'database' AS type, - parent AS label, - NULL AS description, - json_object( - 'method', 'database', - 'database', parent - ) AS url, - parent AS search_text, - NULL AS display_name - FROM allowed_databases - """, - params=database_params, - ), - JumpSQL( - sql=f""" - WITH allowed_tables AS ( - {table_sql} - ) - SELECT - CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, - allowed_tables.parent || ': ' || allowed_tables.child AS label, - NULL AS description, - json_object( - 'method', 'table', - 'database', allowed_tables.parent, - 'table', allowed_tables.child - ) AS url, - allowed_tables.parent || ' ' || allowed_tables.child AS search_text, - NULL AS display_name - FROM allowed_tables - LEFT JOIN catalog_views - ON catalog_views.database_name = allowed_tables.parent - AND catalog_views.view_name = allowed_tables.child - """, - params=table_params, - ), - JumpSQL( - sql=f""" - WITH allowed_queries AS ( - {query_sql} - ) - SELECT - 'query' AS type, - allowed_queries.parent || ': ' || allowed_queries.child AS label, - NULL AS description, - json_object( - 'method', 'query', - 'database', allowed_queries.parent, - 'query', allowed_queries.child - ) AS url, - allowed_queries.parent || ' ' || allowed_queries.child AS search_text, - NULL AS display_name - FROM allowed_queries - """, - params=query_params, - ), - ] - - return inner diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py new file mode 100644 index 00000000..22e6e46a --- /dev/null +++ b/datasette/default_menu_links.py @@ -0,0 +1,41 @@ +from datasette import hookimpl + + +@hookimpl +def menu_links(datasette, actor): + async def inner(): + if not await datasette.permission_allowed(actor, "debug-menu"): + return [] + + return [ + {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, + { + "href": datasette.urls.path("/-/plugins"), + "label": "Installed plugins", + }, + { + "href": datasette.urls.path("/-/versions"), + "label": "Version info", + }, + { + "href": datasette.urls.path("/-/settings"), + "label": "Settings", + }, + { + "href": datasette.urls.path("/-/permissions"), + "label": "Debug permissions", + }, + { + "href": datasette.urls.path("/-/messages"), + "label": "Debug messages", + }, + { + "href": datasette.urls.path("/-/allow-debug"), + "label": "Debug allow rules", + }, + {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"}, + {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, + {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, + ] + + return inner diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py new file mode 100644 index 00000000..a9534cab --- /dev/null +++ b/datasette/default_permissions.py @@ -0,0 +1,577 @@ +from datasette import hookimpl, Permission +from datasette.utils.permissions import PluginSQL +from datasette.utils import actor_matches_allow +import itsdangerous +import time + + +@hookimpl +def register_permissions(): + return ( + Permission( + name="view-instance", + abbr="vi", + description="View Datasette instance", + takes_database=False, + takes_resource=False, + default=True, + ), + Permission( + name="view-database", + abbr="vd", + description="View database", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="view-database-download", + abbr="vdd", + description="Download database file", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="view-table", + abbr="vt", + description="View table", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="view-query", + abbr="vq", + description="View named query results", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="execute-sql", + abbr="es", + description="Execute read-only SQL queries", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="permissions-debug", + abbr="pd", + description="Access permission debug tool", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="debug-menu", + abbr="dm", + description="View debug menu items", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="insert-row", + abbr="ir", + description="Insert rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="delete-row", + abbr="dr", + description="Delete rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="update-row", + abbr="ur", + description="Update rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="create-table", + abbr="ct", + description="Create tables", + takes_database=True, + takes_resource=False, + default=False, + ), + Permission( + name="alter-table", + abbr="at", + description="Alter tables", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="drop-table", + abbr="dt", + description="Drop tables", + takes_database=True, + takes_resource=True, + default=False, + ), + ) + + +@hookimpl(tryfirst=True, specname="permission_allowed") +def permission_allowed_default(datasette, actor, action, resource): + async def inner(): + # id=root gets some special permissions: + if action in ( + "permissions-debug", + "debug-menu", + "insert-row", + "create-table", + "alter-table", + "drop-table", + "delete-row", + "update-row", + ): + if actor and actor.get("id") == "root": + return True + + # Resolve view permissions in allow blocks in configuration + if action in ( + "view-instance", + "view-database", + "view-table", + "view-query", + "execute-sql", + ): + result = await _resolve_config_view_permissions( + datasette, actor, action, resource + ) + if result is not None: + return result + + # Resolve custom permissions: blocks in configuration + result = await _resolve_config_permissions_blocks( + datasette, actor, action, resource + ) + if result is not None: + return result + + # --setting default_allow_sql + if action == "execute-sql" and not datasette.setting("default_allow_sql"): + return False + + return inner + + +@hookimpl +async def permission_resources_sql(datasette, actor, action): + rules: list[PluginSQL] = [] + + config_rules = await _config_permission_rules(datasette, actor, action) + rules.extend(config_rules) + + default_allow_actions = { + "view-instance", + "view-database", + "view-table", + "execute-sql", + } + if action in default_allow_actions: + reason = f"default allow for {action}".replace("'", "''") + sql = ( + "SELECT NULL AS parent, NULL AS child, 1 AS allow, " f"'{reason}' AS reason" + ) + rules.append( + PluginSQL( + source="default_permissions", + sql=sql, + params={}, + ) + ) + + if not rules: + return None + if len(rules) == 1: + return rules[0] + return rules + + +async def _config_permission_rules(datasette, actor, action) -> list[PluginSQL]: + config = datasette.config or {} + + if actor is None: + actor_dict: dict | None = None + elif isinstance(actor, dict): + actor_dict = actor + else: + actor_lookup = await datasette.actors_from_ids([actor]) + actor_dict = actor_lookup.get(actor) or {"id": actor} + + def evaluate(allow_block): + if allow_block is None: + return None + return actor_matches_allow(actor_dict, allow_block) + + rows = [] + + def add_row(parent, child, result, scope): + if result is None: + return + rows.append( + ( + parent, + child, + bool(result), + f"config {'allow' if result else 'deny'} {scope}", + ) + ) + + root_perm = (config.get("permissions") or {}).get(action) + add_row(None, None, evaluate(root_perm), f"permissions for {action}") + + for db_name, db_config in (config.get("databases") or {}).items(): + db_perm = (db_config.get("permissions") or {}).get(action) + add_row( + db_name, None, evaluate(db_perm), f"permissions for {action} on {db_name}" + ) + + for table_name, table_config in (db_config.get("tables") or {}).items(): + table_perm = (table_config.get("permissions") or {}).get(action) + add_row( + db_name, + table_name, + evaluate(table_perm), + f"permissions for {action} on {db_name}/{table_name}", + ) + + if action == "view-table": + table_allow = (table_config or {}).get("allow") + add_row( + db_name, + table_name, + evaluate(table_allow), + f"allow for {action} on {db_name}/{table_name}", + ) + + for query_name, query_config in (db_config.get("queries") or {}).items(): + query_perm = (query_config.get("permissions") or {}).get(action) + add_row( + db_name, + query_name, + evaluate(query_perm), + f"permissions for {action} on {db_name}/{query_name}", + ) + if action == "view-query": + query_allow = (query_config or {}).get("allow") + add_row( + db_name, + query_name, + evaluate(query_allow), + f"allow for {action} on {db_name}/{query_name}", + ) + + if action == "view-database": + db_allow = db_config.get("allow") + add_row( + db_name, None, evaluate(db_allow), f"allow for {action} on {db_name}" + ) + + if action == "execute-sql": + db_allow_sql = db_config.get("allow_sql") + add_row(db_name, None, evaluate(db_allow_sql), f"allow_sql for {db_name}") + + if action == "view-instance": + allow_block = config.get("allow") + add_row(None, None, evaluate(allow_block), "allow for view-instance") + + if action == "view-table": + # Tables handled in loop + pass + + if action == "view-query": + # Queries handled in loop + pass + + if action == "execute-sql": + allow_sql = config.get("allow_sql") + add_row(None, None, evaluate(allow_sql), "allow_sql") + + if action == "view-database": + # already handled per-database + pass + + if not rows: + return [] + + parts = [] + params = {} + for idx, (parent, child, allow, reason) in enumerate(rows): + key = f"cfg_{idx}" + parts.append( + f"SELECT :{key}_parent AS parent, :{key}_child AS child, :{key}_allow AS allow, :{key}_reason AS reason" + ) + params[f"{key}_parent"] = parent + params[f"{key}_child"] = child + params[f"{key}_allow"] = 1 if allow else 0 + params[f"{key}_reason"] = reason + + sql = "\nUNION ALL\n".join(parts) + print(sql, params) + return [PluginSQL(source="config_permissions", sql=sql, params=params)] + + +async def _resolve_config_permissions_blocks(datasette, actor, action, resource): + # Check custom permissions: blocks + config = datasette.config or {} + root_block = (config.get("permissions", None) or {}).get(action) + if root_block: + root_result = actor_matches_allow(actor, root_block) + if root_result is not None: + return root_result + # Now try database-specific blocks + if not resource: + return None + if isinstance(resource, str): + database = resource + else: + database = resource[0] + database_block = ( + (config.get("databases", {}).get(database, {}).get("permissions", None)) or {} + ).get(action) + if database_block: + database_result = actor_matches_allow(actor, database_block) + if database_result is not None: + return database_result + # Finally try table/query specific blocks + if not isinstance(resource, tuple): + return None + database, table_or_query = resource + table_block = ( + ( + config.get("databases", {}) + .get(database, {}) + .get("tables", {}) + .get(table_or_query, {}) + .get("permissions", None) + ) + or {} + ).get(action) + if table_block: + table_result = actor_matches_allow(actor, table_block) + if table_result is not None: + return table_result + # Finally the canned queries + query_block = ( + ( + config.get("databases", {}) + .get(database, {}) + .get("queries", {}) + .get(table_or_query, {}) + .get("permissions", None) + ) + or {} + ).get(action) + if query_block: + query_result = actor_matches_allow(actor, query_block) + if query_result is not None: + return query_result + return None + + +async def _resolve_config_view_permissions(datasette, actor, action, resource): + config = datasette.config or {} + if action == "view-instance": + allow = config.get("allow") + if allow is not None: + return actor_matches_allow(actor, allow) + elif action == "view-database": + database_allow = ((config.get("databases") or {}).get(resource) or {}).get( + "allow" + ) + if database_allow is None: + return None + return actor_matches_allow(actor, database_allow) + elif action == "view-table": + database, table = resource + tables = ((config.get("databases") or {}).get(database) or {}).get( + "tables" + ) or {} + table_allow = (tables.get(table) or {}).get("allow") + if table_allow is None: + return None + return actor_matches_allow(actor, table_allow) + elif action == "view-query": + # Check if this query has a "allow" block in config + database, query_name = resource + query = await datasette.get_canned_query(database, query_name, actor) + assert query is not None + allow = query.get("allow") + if allow is None: + return None + return actor_matches_allow(actor, allow) + elif action == "execute-sql": + # Use allow_sql block from database block, or from top-level + database_allow_sql = ((config.get("databases") or {}).get(resource) or {}).get( + "allow_sql" + ) + if database_allow_sql is None: + database_allow_sql = config.get("allow_sql") + if database_allow_sql is None: + return None + return actor_matches_allow(actor, database_allow_sql) + + +def restrictions_allow_action( + datasette: "Datasette", + restrictions: dict, + action: str, + resource: str | tuple[str, str], +): + "Do these restrictions allow the requested action against the requested resource?" + if action == "view-instance": + # Special case for view-instance: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set + all_rules = restrictions.get("a") or [] + for database_rules in (restrictions.get("d") or {}).values(): + all_rules += database_rules + for database_resource_rules in (restrictions.get("r") or {}).values(): + for resource_rules in database_resource_rules.values(): + all_rules += resource_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view): + return True + + if action == "view-database": + # Special case for view-database: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set AND takes_database + all_rules = restrictions.get("a") or [] + database_rules = list((restrictions.get("d") or {}).get(resource) or []) + all_rules += database_rules + resource_rules = ((restrictions.get("r") or {}).get(resource) or {}).values() + for resource_rules in (restrictions.get("r") or {}).values(): + for table_rules in resource_rules.values(): + all_rules += table_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view and p.takes_database): + return True + + # Does this action have an abbreviation? + to_check = {action} + permission = datasette.permissions.get(action) + if permission and permission.abbr: + to_check.add(permission.abbr) + + # If restrictions is defined then we use those to further restrict the actor + # Crucially, we only use this to say NO (return False) - we never + # use it to return YES (True) because that might over-ride other + # restrictions placed on this actor + all_allowed = restrictions.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if to_check.intersection(all_allowed): + return True + # How about for the current database? + if resource: + if isinstance(resource, str): + database_name = resource + else: + database_name = resource[0] + database_allowed = restrictions.get("d", {}).get(database_name) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if to_check.intersection(database_allowed): + return True + # Or the current table? That's any time the resource is (database, table) + if resource is not None and not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + # TODO: What should this do for canned queries? + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True + + # This action is not specifically allowed, so reject it + return False + + +@hookimpl(specname="permission_allowed") +def permission_allowed_actor_restrictions(datasette, actor, action, resource): + if actor is None: + return None + if "_r" not in actor: + # No restrictions, so we have no opinion + return None + _r = actor.get("_r") + if restrictions_allow_action(datasette, _r, action, resource): + # Return None because we do not have an opinion here + return None + else: + # Block this permission check + return False + + +@hookimpl +def actor_from_request(datasette, request): + prefix = "dstok_" + if not datasette.setting("allow_signed_tokens"): + return None + max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") + authorization = request.headers.get("authorization") + if not authorization: + return None + if not authorization.startswith("Bearer "): + return None + token = authorization[len("Bearer ") :] + if not token.startswith(prefix): + return None + token = token[len(prefix) :] + try: + decoded = datasette.unsign(token, namespace="token") + except itsdangerous.BadSignature: + return None + if "t" not in decoded: + # Missing timestamp + return None + created = decoded["t"] + if not isinstance(created, int): + # Invalid timestamp + return None + duration = decoded.get("d") + if duration is not None and not isinstance(duration, int): + # Invalid duration + return None + if (duration is None and max_signed_tokens_ttl) or ( + duration is not None + and max_signed_tokens_ttl + and duration > max_signed_tokens_ttl + ): + duration = max_signed_tokens_ttl + if duration: + if time.time() - created > duration: + # Expired + return None + actor = {"id": decoded["a"], "token": "dstok"} + if "_r" in decoded: + actor["_r"] = decoded["_r"] + if duration: + actor["token_expires"] = created + duration + return actor + + +@hookimpl +def skip_csrf(scope): + # Skip CSRF check for requests with content-type: application/json + if scope["type"] == "http": + headers = scope.get("headers") or {} + if dict(headers).get(b"content-type") == b"application/json": + return True diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py deleted file mode 100644 index 6cd46f04..00000000 --- a/datasette/default_permissions/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Default permission implementations for Datasette. - -This module provides the built-in permission checking logic through implementations -of the permission_resources_sql hook. The hooks are organized by their purpose: - -1. Actor Restrictions - Enforces _r allowlists embedded in actor tokens -2. Root User - Grants full access when --root flag is used -3. Config Rules - Applies permissions from datasette.yaml -4. Default Settings - Enforces default_allow_sql and default view permissions - -IMPORTANT: These hooks return PermissionSQL objects that are combined using SQL -UNION/INTERSECT operations. The order of evaluation is: - - restriction_sql fields are INTERSECTed (all must match) - - Regular sql fields are UNIONed and evaluated with cascading priority -""" - -from __future__ import annotations - -# Re-export all hooks and public utilities -from .restrictions import ( - actor_restrictions_sql as actor_restrictions_sql, - restrictions_allow_action as restrictions_allow_action, - ActorRestrictions as ActorRestrictions, -) -from .root import root_user_permissions_sql as root_user_permissions_sql -from .config import config_permissions_sql as config_permissions_sql -from .defaults import ( - # Avoid "datasette.default_permissions" does not explicitly export attribute - default_allow_sql_check as default_allow_sql_check, - default_action_permissions_sql as default_action_permissions_sql, - default_query_permissions_sql as default_query_permissions_sql, - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, -) diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py deleted file mode 100644 index aab87c1c..00000000 --- a/datasette/default_permissions/config.py +++ /dev/null @@ -1,442 +0,0 @@ -""" -Config-based permission handling for Datasette. - -Applies permission rules from datasette.yaml configuration. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL -from datasette.utils import actor_matches_allow - -from .helpers import PermissionRowCollector, get_action_name_variants - - -class ConfigPermissionProcessor: - """ - Processes permission rules from datasette.yaml configuration. - - Configuration structure: - - permissions: # Root-level permissions block - view-instance: - id: admin - - databases: - mydb: - permissions: # Database-level permissions - view-database: - id: admin - allow: # Database-level allow block (for view-*) - id: viewer - allow_sql: # execute-sql allow block - id: analyst - tables: - users: - permissions: # Table-level permissions - view-table: - id: admin - allow: # Table-level allow block - id: viewer - queries: - my_query: - permissions: # Query-level permissions - view-query: - id: admin - allow: # Query-level allow block - id: viewer - """ - - def __init__( - self, - datasette: "Datasette", - actor: Optional[dict], - action: str, - ): - self.datasette = datasette - self.actor = actor - self.action = action - self.config = datasette.config or {} - self.collector = PermissionRowCollector(prefix="cfg") - - # Pre-compute action variants - self.action_checks = get_action_name_variants(datasette, action) - self.action_obj = datasette.actions.get(action) - - # Parse restrictions if present - self.has_restrictions = actor and "_r" in actor if actor else False - self.restrictions = actor.get("_r", {}) if actor else {} - - # Pre-compute restriction info for efficiency - self.restricted_databases: Set[str] = set() - self.restricted_tables: Set[Tuple[str, str]] = set() - - if self.has_restrictions: - self.restricted_databases = { - db_name - for db_name, db_actions in (self.restrictions.get("d") or {}).items() - if self.action_checks.intersection(db_actions) - } - self.restricted_tables = { - (db_name, table_name) - for db_name, tables in (self.restrictions.get("r") or {}).items() - for table_name, table_actions in tables.items() - if self.action_checks.intersection(table_actions) - } - # Tables implicitly reference their parent databases - self.restricted_databases.update(db for db, _ in self.restricted_tables) - - def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: - """Evaluate an allow block against the current actor.""" - if allow_block is None: - return None - return actor_matches_allow(self.actor, allow_block) - - def is_in_restriction_allowlist( - self, - parent: Optional[str], - child: Optional[str], - ) -> bool: - """Check if resource is allowed by actor restrictions.""" - if not self.has_restrictions: - return True # No restrictions, all resources allowed - - # Check global allowlist - if self.action_checks.intersection(self.restrictions.get("a", [])): - return True - - # Check database-level allowlist - if parent and self.action_checks.intersection( - self.restrictions.get("d", {}).get(parent, []) - ): - return True - - # Check table-level allowlist - if parent: - table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) - if child: - table_actions = table_restrictions.get(child, []) - if self.action_checks.intersection(table_actions): - return True - else: - # Parent query should proceed if any child in this database is allowlisted - for table_actions in table_restrictions.values(): - if self.action_checks.intersection(table_actions): - return True - - # Parent/child both None: include if any restrictions exist for this action - if parent is None and child is None: - if self.action_checks.intersection(self.restrictions.get("a", [])): - return True - if self.restricted_databases: - return True - if self.restricted_tables: - return True - - return False - - def add_permissions_rule( - self, - parent: Optional[str], - child: Optional[str], - permissions_block: Optional[dict], - scope_desc: str, - ) -> None: - """Add a rule from a permissions:{action} block.""" - if permissions_block is None: - return - - action_allow_block = permissions_block.get(self.action) - result = self.evaluate_allow_block(action_allow_block) - - self.collector.add( - parent=parent, - child=child, - allow=result, - reason=f"config {'allow' if result else 'deny'} {scope_desc}", - if_not_none=True, - ) - - def add_allow_block_rule( - self, - parent: Optional[str], - child: Optional[str], - allow_block: Any, - scope_desc: str, - ) -> None: - """ - Add rules from an allow:{} block. - - For allow blocks, if the block exists but doesn't match the actor, - this is treated as a deny. We also handle the restriction-gate logic. - """ - if allow_block is None: - return - - # Skip if resource is not in restriction allowlist - if not self.is_in_restriction_allowlist(parent, child): - return - - result = self.evaluate_allow_block(allow_block) - bool_result = bool(result) - - self.collector.add( - parent, - child, - bool_result, - f"config {'allow' if result else 'deny'} {scope_desc}", - ) - - # Handle restriction-gate: add explicit denies for restricted resources - self._add_restriction_gate_denies(parent, child, bool_result, scope_desc) - - def _add_restriction_gate_denies( - self, - parent: Optional[str], - child: Optional[str], - is_allowed: bool, - scope_desc: str, - ) -> None: - """ - When a config rule denies at a higher level, add explicit denies - for restricted resources to prevent child-level allows from - incorrectly granting access. - """ - if is_allowed or child is not None or not self.has_restrictions: - return - - if not self.action_obj: - return - - reason = f"config deny {scope_desc} (restriction gate)" - - if parent is None: - # Root-level deny: add denies for all restricted resources - if self.action_obj.takes_parent: - for db_name in self.restricted_databases: - self.collector.add(db_name, None, False, reason) - if self.action_obj.takes_child: - for db_name, table_name in self.restricted_tables: - self.collector.add(db_name, table_name, False, reason) - else: - # Database-level deny: add denies for tables in that database - if self.action_obj.takes_child: - for db_name, table_name in self.restricted_tables: - if db_name == parent: - self.collector.add(db_name, table_name, False, reason) - - def process(self) -> Optional[PermissionSQL]: - """Process all config rules and return combined PermissionSQL.""" - self._process_root_permissions() - self._process_databases() - self._process_root_allow_blocks() - - return self.collector.to_permission_sql() - - def _process_root_permissions(self) -> None: - """Process root-level permissions block.""" - root_perms = self.config.get("permissions") or {} - self.add_permissions_rule( - None, - None, - root_perms, - f"permissions for {self.action}", - ) - - def _process_databases(self) -> None: - """Process database-level and nested configurations.""" - databases = self.config.get("databases") or {} - - for db_name, db_config in databases.items(): - self._process_database(db_name, db_config or {}) - - def _process_database(self, db_name: str, db_config: dict) -> None: - """Process a single database's configuration.""" - # Database-level permissions block - db_perms = db_config.get("permissions") or {} - self.add_permissions_rule( - db_name, - None, - db_perms, - f"permissions for {self.action} on {db_name}", - ) - - # Process tables - for table_name, table_config in (db_config.get("tables") or {}).items(): - self._process_table(db_name, table_name, table_config or {}) - - # Process queries - for query_name, query_config in (db_config.get("queries") or {}).items(): - self._process_query(db_name, query_name, query_config) - - # Database-level allow blocks - self._process_database_allow_blocks(db_name, db_config) - - def _process_table( - self, - db_name: str, - table_name: str, - table_config: dict, - ) -> None: - """Process a single table's configuration.""" - # Table-level permissions block - table_perms = table_config.get("permissions") or {} - self.add_permissions_rule( - db_name, - table_name, - table_perms, - f"permissions for {self.action} on {db_name}/{table_name}", - ) - - # Table-level allow block (for view-table) - if self.action == "view-table": - self.add_allow_block_rule( - db_name, - table_name, - table_config.get("allow"), - f"allow for {self.action} on {db_name}/{table_name}", - ) - - def _process_query( - self, - db_name: str, - query_name: str, - query_config: Any, - ) -> None: - """Process a single query's configuration.""" - # Query config can be a string (just SQL) or dict - if not isinstance(query_config, dict): - return - - # Query-level permissions block - query_perms = query_config.get("permissions") or {} - self.add_permissions_rule( - db_name, - query_name, - query_perms, - f"permissions for {self.action} on {db_name}/{query_name}", - ) - - # Query-level allow block (for view-query) - if self.action == "view-query": - self.add_allow_block_rule( - db_name, - query_name, - query_config.get("allow"), - f"allow for {self.action} on {db_name}/{query_name}", - ) - - def _process_database_allow_blocks( - self, - db_name: str, - db_config: dict, - ) -> None: - """Process database-level allow/allow_sql blocks.""" - # view-database allow block - if self.action == "view-database": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow"), - f"allow for {self.action} on {db_name}", - ) - - # execute-sql allow_sql block - if self.action == "execute-sql": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow_sql"), - f"allow_sql for {db_name}", - ) - - # view-table uses database-level allow for inheritance - if self.action == "view-table": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow"), - f"allow for {self.action} on {db_name}", - ) - - # view-query uses database-level allow for inheritance - if self.action == "view-query": - self.add_allow_block_rule( - db_name, - None, - db_config.get("allow"), - f"allow for {self.action} on {db_name}", - ) - - def _process_root_allow_blocks(self) -> None: - """Process root-level allow/allow_sql blocks.""" - root_allow = self.config.get("allow") - - if self.action == "view-instance": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-instance", - ) - - if self.action == "view-database": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-database", - ) - - if self.action == "view-table": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-table", - ) - - if self.action == "view-query": - self.add_allow_block_rule( - None, - None, - root_allow, - "allow for view-query", - ) - - if self.action == "execute-sql": - self.add_allow_block_rule( - None, - None, - self.config.get("allow_sql"), - "allow_sql", - ) - - -@hookimpl(specname="permission_resources_sql") -async def config_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[List[PermissionSQL]]: - """ - Apply permission rules from datasette.yaml configuration. - - This processes: - - permissions: blocks at root, database, table, and query levels - - allow: blocks for view-* actions - - allow_sql: blocks for execute-sql action - """ - processor = ConfigPermissionProcessor(datasette, actor, action) - result = processor.process() - - if result is None: - return [] - - return [result] diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py deleted file mode 100644 index 5bc74425..00000000 --- a/datasette/default_permissions/defaults.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -Default permission settings for Datasette. - -Provides default allow rules for standard view/execute actions. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - -# Actions that are allowed by default (unless --default-deny is used) -DEFAULT_ALLOW_ACTIONS = frozenset( - { - "view-instance", - "view-database", - "view-database-download", - "view-table", - "view-query", - "execute-sql", - } -) - - -@hookimpl(specname="permission_resources_sql") -async def default_allow_sql_check( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - """ - Enforce the default_allow_sql setting. - - When default_allow_sql is false (the default), execute-sql is denied - unless explicitly allowed by config or other rules. - """ - if action == "execute-sql": - if not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") - - return None - - -@hookimpl(specname="permission_resources_sql") -async def default_action_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - """ - Provide default allow rules for standard view/execute actions. - - These defaults are skipped when datasette is started with --default-deny. - The restriction_sql mechanism (from actor_restrictions_sql) will still - filter these results if the actor has restrictions. - """ - if datasette.default_deny: - return None - - if action in DEFAULT_ALLOW_ACTIONS: - reason = f"default allow for {action}".replace("'", "''") - return PermissionSQL.allow(reason=reason) - - return None - - -@hookimpl(specname="permission_resources_sql") -async def default_query_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - actor_id = actor.get("id") if isinstance(actor, dict) else None - - if action not in {"view-query", "update-query", "delete-query"}: - return None - - params = {"query_owner_id": actor_id} - rule_sqls = [] - if actor_id is not None: - if action in {"update-query", "delete-query"}: - # Query owner can update/delete query - rule_sqls.append(""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE source = 'user' - AND owner_id = :query_owner_id - """) - else: - # Query owner can view-query - rule_sqls.append(""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE owner_id = :query_owner_id - """) - - # restriction_sql enforces private queries ONLY visible/mutable by owner - return PermissionSQL( - sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, - restriction_sql=""" - SELECT database_name AS parent, name AS child - FROM queries - WHERE is_private = 0 - OR owner_id = :query_owner_id - """, - params=params, - ) diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py deleted file mode 100644 index 47e03569..00000000 --- a/datasette/default_permissions/helpers.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Shared helper utilities for default permission implementations. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette.permissions import PermissionSQL - - -def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: - """ - Get all name variants for an action (full name and abbreviation). - - Example: - get_action_name_variants(ds, "view-table") -> {"view-table", "vt"} - """ - variants = {action} - action_obj = datasette.actions.get(action) - if action_obj and action_obj.abbr: - variants.add(action_obj.abbr) - return variants - - -def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: - """Check if an action (or its abbreviation) is in a list.""" - return bool(get_action_name_variants(datasette, action).intersection(action_list)) - - -@dataclass -class PermissionRow: - """A single permission rule row.""" - - parent: Optional[str] - child: Optional[str] - allow: bool - reason: str - - -class PermissionRowCollector: - """Collects permission rows and converts them to PermissionSQL.""" - - def __init__(self, prefix: str = "row"): - self.rows: List[PermissionRow] = [] - self.prefix = prefix - - def add( - self, - parent: Optional[str], - child: Optional[str], - allow: Optional[bool], - reason: str, - if_not_none: bool = False, - ) -> None: - """Add a permission row. If if_not_none=True, only add if allow is not None.""" - if if_not_none and allow is None: - return - self.rows.append(PermissionRow(parent, child, allow, reason)) - - def to_permission_sql(self) -> Optional[PermissionSQL]: - """Convert collected rows to a PermissionSQL object.""" - if not self.rows: - return None - - parts = [] - params = {} - - for idx, row in enumerate(self.rows): - key = f"{self.prefix}_{idx}" - parts.append( - f"SELECT :{key}_parent AS parent, :{key}_child AS child, " - f":{key}_allow AS allow, :{key}_reason AS reason" - ) - params[f"{key}_parent"] = row.parent - params[f"{key}_child"] = row.child - params[f"{key}_allow"] = 1 if row.allow else 0 - params[f"{key}_reason"] = row.reason - - sql = "\nUNION ALL\n".join(parts) - return PermissionSQL(sql=sql, params=params) diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py deleted file mode 100644 index a22cd7e5..00000000 --- a/datasette/default_permissions/restrictions.py +++ /dev/null @@ -1,195 +0,0 @@ -""" -Actor restriction handling for Datasette permissions. - -This module handles the _r (restrictions) key in actor dictionaries, which -contains allowlists of resources the actor can access. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set, Tuple - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - -from .helpers import action_in_list, get_action_name_variants - - -@dataclass -class ActorRestrictions: - """Parsed actor restrictions from the _r key.""" - - global_actions: List[str] # _r.a - globally allowed actions - database_actions: dict # _r.d - {db_name: [actions]} - table_actions: dict # _r.r - {db_name: {table: [actions]}} - - @classmethod - def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: - """Parse restrictions from actor dict. Returns None if no restrictions.""" - if not actor: - return None - assert isinstance(actor, dict), "actor must be a dictionary" - - restrictions = actor.get("_r") - if restrictions is None: - return None - - return cls( - global_actions=restrictions.get("a", []), - database_actions=restrictions.get("d", {}), - table_actions=restrictions.get("r", {}), - ) - - def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: - """Check if action is in the global allowlist.""" - return action_in_list(datasette, action, self.global_actions) - - def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: - """Get database names where this action is allowed.""" - allowed = set() - for db_name, db_actions in self.database_actions.items(): - if action_in_list(datasette, action, db_actions): - allowed.add(db_name) - return allowed - - def get_allowed_tables( - self, datasette: "Datasette", action: str - ) -> Set[Tuple[str, str]]: - """Get (database, table) pairs where this action is allowed.""" - allowed = set() - for db_name, tables in self.table_actions.items(): - for table_name, table_actions in tables.items(): - if action_in_list(datasette, action, table_actions): - allowed.add((db_name, table_name)) - return allowed - - -@hookimpl(specname="permission_resources_sql") -async def actor_restrictions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[List[PermissionSQL]]: - """ - Handle actor restriction-based permission rules. - - When an actor has an "_r" key, it contains an allowlist of resources they - can access. This function returns restriction_sql that filters the final - results to only include resources in that allowlist. - - The _r structure: - { - "a": ["vi", "pd"], # Global actions allowed - "d": {"mydb": ["vt", "es"]}, # Database-level actions - "r": {"mydb": {"users": ["vt"]}} # Table-level actions - } - """ - if not actor: - return None - - restrictions = ActorRestrictions.from_actor(actor) - - if restrictions is None: - # No restrictions - all resources allowed - return [] - - # If globally allowed, no filtering needed - if restrictions.is_action_globally_allowed(datasette, action): - return [] - - # Build restriction SQL - allowed_dbs = restrictions.get_allowed_databases(datasette, action) - allowed_tables = restrictions.get_allowed_tables(datasette, action) - - # If nothing is allowed for this action, return empty-set restriction - if not allowed_dbs and not allowed_tables: - return [ - PermissionSQL( - params={"deny": f"actor restrictions: {action} not in allowlist"}, - restriction_sql="SELECT NULL AS parent, NULL AS child WHERE 0", - ) - ] - - # Build UNION of allowed resources - selects = [] - params = {} - counter = 0 - - # Database-level entries (parent, NULL) - allows all children - for db_name in allowed_dbs: - key = f"restr_{counter}" - counter += 1 - selects.append(f"SELECT :{key}_parent AS parent, NULL AS child") - params[f"{key}_parent"] = db_name - - # Table-level entries (parent, child) - for db_name, table_name in allowed_tables: - key = f"restr_{counter}" - counter += 1 - selects.append(f"SELECT :{key}_parent AS parent, :{key}_child AS child") - params[f"{key}_parent"] = db_name - params[f"{key}_child"] = table_name - - restriction_sql = "\nUNION ALL\n".join(selects) - - return [PermissionSQL(params=params, restriction_sql=restriction_sql)] - - -def restrictions_allow_action( - datasette: "Datasette", - restrictions: dict, - action: str, - resource: Optional[str | Tuple[str, str]], -) -> bool: - """ - Check if restrictions allow the requested action on the requested resource. - - This is a synchronous utility function for use by other code that needs - to quickly check restriction allowlists. - - Args: - datasette: The Datasette instance - restrictions: The _r dict from an actor - action: The action name to check - resource: None for global, str for database, (db, table) tuple for table - - Returns: - True if allowed, False if denied - """ - # Does this action have an abbreviation? - to_check = get_action_name_variants(datasette, action) - - # Check global level (any resource) - all_allowed = restrictions.get("a") - if all_allowed is not None: - assert isinstance(all_allowed, list) - if to_check.intersection(all_allowed): - return True - - # Check database level - if resource: - if isinstance(resource, str): - database_name = resource - else: - database_name = resource[0] - database_allowed = restrictions.get("d", {}).get(database_name) - if database_allowed is not None: - assert isinstance(database_allowed, list) - if to_check.intersection(database_allowed): - return True - - # Check table/resource level - if resource is not None and not isinstance(resource, str) and len(resource) == 2: - database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True - - # This action is not explicitly allowed, so reject it - return False diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py deleted file mode 100644 index 4931f7ff..00000000 --- a/datasette/default_permissions/root.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Root user permission handling for Datasette. - -Grants full permissions to the root user when --root flag is used. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - - -@hookimpl(specname="permission_resources_sql") -async def root_user_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], -) -> Optional[PermissionSQL]: - """ - Grant root user full permissions when --root flag is used. - """ - if not datasette.root_enabled: - return None - if actor is not None and actor.get("id") == "root": - return PermissionSQL.allow(reason="root user") diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py deleted file mode 100644 index 7a359dc6..00000000 --- a/datasette/default_permissions/tokens.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Token authentication for Datasette. - -Registers the default SignedTokenHandler and delegates token verification -to datasette.verify_token() so all registered handlers are tried. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl -from datasette.tokens import SignedTokenHandler - - -@hookimpl -def register_token_handler(datasette: "Datasette"): - """Register the default signed token handler.""" - return SignedTokenHandler() - - -@hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token( - datasette: "Datasette", request -) -> Optional[dict]: - """ - Authenticate requests using API tokens by delegating to all registered - token handlers via datasette.verify_token(). - """ - authorization = request.headers.get("authorization") - if not authorization: - return None - if not authorization.startswith("Bearer "): - return None - - token = authorization[len("Bearer ") :] - return await datasette.verify_token(token) diff --git a/datasette/events.py b/datasette/events.py index e8786da9..ae90972d 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -2,6 +2,7 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field from datasette.hookspecs import hookimpl from datetime import datetime, timezone +from typing import Optional @dataclass @@ -13,7 +14,7 @@ class Event(ABC): created: datetime = field( init=False, default_factory=lambda: datetime.now(timezone.utc) ) - actor: dict | None + actor: Optional[dict] def properties(self): properties = asdict(self) @@ -62,7 +63,7 @@ class CreateTokenEvent(Event): """ name = "create-token" - expires_after: int | None + expires_after: Optional[int] restrict_all: list restrict_database: dict restrict_resource: dict @@ -199,27 +200,6 @@ class UpdateRowEvent(Event): pks: list -@dataclass -class RenameTableEvent(Event): - """ - Event name: ``rename-table`` - - A table has been renamed. - - :ivar database: The name of the database containing the renamed table. - :type database: str - :ivar old_table: The previous name of the table. - :type old_table: str - :ivar new_table: The new name of the table. - :type new_table: str - """ - - name = "rename-table" - database: str - old_table: str - new_table: str - - @dataclass class DeleteRowEvent(Event): """ @@ -240,42 +220,6 @@ class DeleteRowEvent(Event): pks: list -@hookimpl -def write_wrapper(datasette, database, request, transaction): - def wrapper(conn, track_event): - # Snapshot rootpage -> name before the write - before = { - row[1]: row[0] - for row in conn.execute( - "select name, rootpage from sqlite_master" - " where type='table' and rootpage != 0" - ).fetchall() - } - yield - # Snapshot rootpage -> name after the write - after = { - row[1]: row[0] - for row in conn.execute( - "select name, rootpage from sqlite_master" - " where type='table' and rootpage != 0" - ).fetchall() - } - # Detect renames: same rootpage, different name - for rootpage, old_name in before.items(): - new_name = after.get(rootpage) - if new_name and new_name != old_name: - track_event( - RenameTableEvent( - actor=request.actor if request else None, - database=database, - old_table=old_name, - new_table=new_name, - ) - ) - - return wrapper - - @hookimpl def register_events(): return [ @@ -284,7 +228,6 @@ def register_events(): CreateTableEvent, CreateTokenEvent, AlterTableEvent, - RenameTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..dd149424 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -83,7 +83,7 @@ class Facet: self.ds = ds self.request = request self.database = database - # For foreign key expansion. Can be None for e.g. stored SQL queries: + # For foreign key expansion. Can be None for e.g. canned SQL queries: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] @@ -233,7 +233,9 @@ class ColumnFacet(Facet): ) where {col} is not null group by {col} order by count desc, value limit {limit} - """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1) + """.format( + col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 + ) try: facet_rows_results = await self.ds.execute( self.database, @@ -480,7 +482,9 @@ class DateFacet(Facet): select date({column}) from ( select * from ({sql}) limit 100 ) where {column} glob "????-??-*" - """.format(column=escape_sqlite(column), sql=self.sql) + """.format( + column=escape_sqlite(column), sql=self.sql + ) try: results = await self.ds.execute( self.database, @@ -526,7 +530,9 @@ class DateFacet(Facet): ) where date({col}) is not null group by date({col}) order by count desc, value limit {limit} - """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1) + """.format( + col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 + ) try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 95cc5f37..67d4170b 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,8 +1,8 @@ from datasette import hookimpl -from datasette.resources import DatabaseResource from datasette.views.base import DatasetteError from datasette.utils.asgi import BadRequest import json +import numbers from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -13,10 +13,11 @@ def where_filters(request, database, datasette): where_clauses = [] extra_wheres_for_ui = [] if "_where" in request.args: - if not await datasette.allowed( - action="execute-sql", - resource=DatabaseResource(database=database), - actor=request.actor, + if not await datasette.permission_allowed( + request.actor, + "execute-sql", + resource=database, + default=True, ): raise DatasetteError("_where= is not allowed", status=403) else: diff --git a/datasette/fixtures.py b/datasette/fixtures.py deleted file mode 100644 index 7c85e16a..00000000 --- a/datasette/fixtures.py +++ /dev/null @@ -1,415 +0,0 @@ -from datasette.utils.sqlite import sqlite3 -from datasette.utils import documented -import itertools -import random -import string - -__all__ = [ - "EXTRA_DATABASE_SQL", - "TABLES", - "TABLE_PARAMETERIZED_SQL", - "generate_compound_rows", - "generate_sortable_rows", - "populate_extra_database", - "populate_fixture_database", - "write_extra_database", - "write_fixture_database", -] - - -def generate_compound_rows(num): - """Generate rows for the compound_three_primary_keys fixture table.""" - for a, b, c in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=3), num - ): - yield a, b, c, f"{a}-{b}-{c}" - - -def generate_sortable_rows(num): - """Generate rows for the sortable fixture table.""" - rand = random.Random(42) - for a, b in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=2), num - ): - yield { - "pk1": a, - "pk2": b, - "content": f"{a}-{b}", - "sortable": rand.randint(-100, 100), - "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), - "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), - "text": rand.choice(["$null", "$blah"]), - } - - -TABLES = ( - """ -CREATE TABLE simple_primary_key ( - id integer primary key, - content text -); - -CREATE TABLE primary_key_multiple_columns ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE primary_key_multiple_columns_explicit_label ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE compound_primary_key ( - pk1 varchar(30), - pk2 varchar(30), - content text, - PRIMARY KEY (pk1, pk2) -); - -INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); -INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); -INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); - -CREATE TABLE compound_three_primary_keys ( - pk1 varchar(30), - pk2 varchar(30), - pk3 varchar(30), - content text, - PRIMARY KEY (pk1, pk2, pk3) -); -CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); - -CREATE TABLE foreign_key_references ( - pk varchar(30) primary key, - foreign_key_with_label integer, - foreign_key_with_blank_label integer, - foreign_key_with_no_label varchar(30), - foreign_key_compound_pk1 varchar(30), - foreign_key_compound_pk2 varchar(30), - FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) - FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) -); - -CREATE TABLE sortable ( - pk1 varchar(30), - pk2 varchar(30), - content text, - sortable integer, - sortable_with_nulls real, - sortable_with_nulls_2 real, - text text, - PRIMARY KEY (pk1, pk2) -); - -CREATE TABLE no_primary_key ( - content text, - a text, - b text, - c text -); - -CREATE TABLE [123_starts_with_digits] ( - content text -); - -CREATE VIEW paginated_view AS - SELECT - content, - '- ' || content || ' -' AS content_extra - FROM no_primary_key; - -CREATE TABLE "Table With Space In Name" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "table/with/slashes.csv" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "complex_foreign_keys" ( - pk varchar(30) primary key, - f1 integer, - f2 integer, - f3 integer, - FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) -); - -CREATE TABLE "custom_foreign_key_label" ( - pk varchar(30) primary key, - foreign_key_with_custom_label text, - FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) -); - -CREATE TABLE tags ( - tag TEXT PRIMARY KEY -); - -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text, - [name with . and spaces] text -); - -CREATE TABLE searchable_tags ( - searchable_id integer, - tag text, - PRIMARY KEY (searchable_id, tag), - FOREIGN KEY (searchable_id) REFERENCES searchable(pk), - FOREIGN KEY (tag) REFERENCES tags(tag) -); - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); - -INSERT INTO tags VALUES ("canine"); -INSERT INTO tags VALUES ("feline"); - -INSERT INTO searchable_tags (searchable_id, tag) VALUES - (1, "feline"), - (2, "canine") -; - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); -INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); - -CREATE TABLE [select] ( - [group] text, - [having] text, - [and] text, - [json] text -); -INSERT INTO [select] VALUES ('group', 'having', 'and', - '{"href": "http://example.com/", "label":"Example"}' -); - -CREATE TABLE infinity ( - value REAL -); -INSERT INTO infinity VALUES - (1e999), - (-1e999), - (1.5) -; - -CREATE TABLE facet_cities ( - id integer primary key, - name text -); -INSERT INTO facet_cities (id, name) VALUES - (1, 'San Francisco'), - (2, 'Los Angeles'), - (3, 'Detroit'), - (4, 'Memnonia') -; - -CREATE TABLE facetable ( - pk integer primary key, - created text, - planet_int integer, - on_earth integer, - state text, - _city_id integer, - _neighborhood text, - tags text, - complex_array text, - distinct_some_null, - n text, - FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) -); -INSERT INTO facetable - (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) -VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) -; - -CREATE TABLE binary_data ( - data BLOB -); - --- Many 2 Many demo: roadside attractions! - -CREATE TABLE roadside_attractions ( - pk integer primary key, - name text, - address text, - url text, - latitude real, - longitude real -); -INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", - 37.0167, -122.0024 -); -INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", - 37.3184, -121.9511 -); -INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, - 37.5793, -122.3442 -); -INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", - 37.0414, -122.0725 -); - -CREATE TABLE attraction_characteristic ( - pk integer primary key, - name text -); -INSERT INTO attraction_characteristic VALUES ( - 1, "Museum" -); -INSERT INTO attraction_characteristic VALUES ( - 2, "Paranormal" -); - -CREATE TABLE roadside_attraction_characteristics ( - attraction_id INTEGER REFERENCES roadside_attractions(pk), - characteristic_id INTEGER REFERENCES attraction_characteristic(pk) -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 1, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 2, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 3, 1 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 1 -); - -INSERT INTO simple_primary_key VALUES (1, 'hello'); -INSERT INTO simple_primary_key VALUES (2, 'world'); -INSERT INTO simple_primary_key VALUES (3, ''); -INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); -INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); - -INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); -INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); - -INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); -INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); - -INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); -INSERT INTO custom_foreign_key_label VALUES (1, 1); - -INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); - -CREATE VIEW simple_view AS - SELECT content, upper(content) AS upper_content FROM simple_primary_key; - -CREATE VIEW searchable_view AS - SELECT * from searchable; - -CREATE VIEW searchable_view_configured_by_metadata AS - SELECT * from searchable; - -""" - + "\n".join( - [ - 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( - i=i + 1 - ) - for i in range(201) - ] - ) - + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' - + "\n".join( - [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) - for a, b, c, content in generate_compound_rows(1001) - ] - ) - + "\n".join(["""INSERT INTO sortable VALUES ( - "{pk1}", "{pk2}", "{content}", {sortable}, - {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); - """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) -) - -TABLE_PARAMETERIZED_SQL = [ - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (null);", []), -] - -EXTRA_DATABASE_SQL = """ -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text -); - -CREATE VIEW searchable_view AS SELECT * FROM searchable; - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS3 (text1, text2, content="searchable"); -INSERT INTO "searchable_fts" (rowid, text1, text2) - SELECT rowid, text1, text2 FROM searchable; -""" - - -@documented(label="datasette_fixtures_populate_fixture_database") -def populate_fixture_database(conn): - """Populate a SQLite connection with Datasette's test fixture tables.""" - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - - -def populate_extra_database(conn): - """Populate a SQLite connection with the extra database used in tests.""" - conn.executescript(EXTRA_DATABASE_SQL) - - -def write_fixture_database(db_filename): - """Write Datasette's test fixture tables to a SQLite database file.""" - conn = sqlite3.connect(db_filename) - try: - populate_fixture_database(conn) - finally: - conn.close() - - -def write_extra_database(db_filename): - """Write the extra test database tables to a SQLite database file.""" - conn = sqlite3.connect(db_filename) - try: - populate_extra_database(conn) - finally: - conn.close() diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index dcd502af..eedb2481 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -55,17 +55,7 @@ def publish_subcommand(publish): @hookspec -def render_cell( - row, - value, - column, - table, - pks, - database, - datasette, - request, - column_type, -): +def render_cell(row, value, column, table, database, datasette, request): """Customize rendering of HTML table cell values""" @@ -80,13 +70,8 @@ def register_facet_classes(): @hookspec -def register_actions(datasette): - """Register actions: returns a list of datasette.permission.Action objects""" - - -@hookspec -def register_column_types(datasette): - """Return a list of ColumnType subclasses""" +def register_permissions(datasette): + """Register permissions: returns a list of datasette.permission.Permission named tuples""" @hookspec @@ -125,18 +110,28 @@ def filters_from_request(request, database, table, datasette): ) based on the request""" +@hookspec +def permission_allowed(datasette, actor, action, resource): + """Check if actor is allowed to perform this action - return True, False or None""" + + @hookspec def permission_resources_sql(datasette, actor, action): """Return SQL query fragments for permission checks on resources. - Returns None, a PermissionSQL object, or a list of PermissionSQL objects. - Each PermissionSQL contains SQL that should return rows with columns: + Returns None, a PluginSQL object, or a list of PluginSQL objects. + Each PluginSQL contains SQL that should return rows with columns: parent (str|None), child (str|None), allow (int), reason (str). Used to efficiently check permissions across multiple resources at once. """ +@hookspec +def canned_queries(datasette, database, actor): + """Return a dictionary of canned query definitions or an awaitable function that returns them""" + + @hookspec def register_magic_parameters(datasette): """Return a list of (name, function) magic parameter functions""" @@ -152,11 +147,6 @@ def menu_links(datasette, actor, request): """Links for the navigation menu""" -@hookspec -def jump_items_sql(datasette, actor, request): - """SQL fragments for extra items in the jump menu""" - - @hookspec def row_actions(datasette, actor, request, database, table, row): """Links for the row actions menu""" @@ -174,7 +164,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and stored query actions menu""" + """Links for the query and canned query actions menu""" @hookspec @@ -187,6 +177,11 @@ def homepage_actions(datasette, actor, request): """Links for the homepage actions menu""" +@hookspec +def skip_csrf(datasette, scope): + """Mechanism for skipping CSRF checks for certain requests""" + + @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" @@ -228,38 +223,5 @@ def top_query(datasette, request, database, sql): @hookspec -def top_stored_query(datasette, request, database, query_name): - """HTML to include at the top of the stored query page""" - - -@hookspec -def register_token_handler(datasette): - """Return a TokenHandler instance for token creation and verification""" - - -@hookspec -def write_wrapper(datasette, database, request, transaction): - """Called when a write function is about to execute. - - Return a generator function that accepts a ``conn`` argument and - optionally a ``track_event`` argument. The generator should - ``yield`` exactly once: code before the ``yield`` runs before - the write, code after the ``yield`` runs after the write - completes. The result of the write is sent back through the - ``yield``, so you can capture it with ``result = yield``. - - If your generator accepts ``track_event``, you can call - ``track_event(event)`` to queue an event that will be dispatched - via ``datasette.track_event()`` after the write commits - successfully. Events are discarded if the write raises an - exception. - - If the write raises an exception, it is thrown into the generator - so you can handle it with a try/except around the ``yield``. - - ``request`` may be ``None`` for writes not originating from an - HTTP request. ``transaction`` is ``True`` if the write will - be wrapped in a transaction. - - Return ``None`` to skip wrapping. - """ +def top_canned_query(datasette, request, database, query_name): + """HTML to include at the top of the canned query page""" diff --git a/datasette/inspect.py b/datasette/inspect.py index 5e681e03..ede142d0 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -10,6 +10,7 @@ from .utils import ( sqlite3, ) + HASH_BLOCK_SIZE = 1024 * 1024 @@ -69,11 +70,16 @@ def inspect_tables(conn, database_metadata): tables[table]["foreign_keys"] = info # Mark tables 'hidden' if they relate to FTS virtual tables - hidden_tables = [r["name"] for r in conn.execute(""" + hidden_tables = [ + r["name"] + for r in conn.execute( + """ select name from sqlite_master where rootpage = 0 and sql like '%VIRTUAL TABLE%USING FTS%' - """)] + """ + ) + ] if detect_spatialite(conn): # Also hide Spatialite internal tables @@ -88,11 +94,14 @@ def inspect_tables(conn, database_metadata): "views_geometry_columns", "virts_geometry_columns", ] + [ - r["name"] for r in conn.execute(""" + r["name"] + for r in conn.execute( + """ select name from sqlite_master where name like "idx_%" and type = "table" - """) + """ + ) ] for t in tables.keys(): diff --git a/datasette/jump.py b/datasette/jump.py deleted file mode 100644 index d138e827..00000000 --- a/datasette/jump.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Any - - -@dataclass -class JumpSQL: - sql: str - params: dict[str, Any] | None = None - database: str | None = None - - @classmethod - def menu_item( - cls, - *, - label: str, - url: str, - description: str = "Menu item", - search_text: str | None = None, - display_name: str | None = None, - item_type: str = "menu", - ) -> "JumpSQL": - if search_text is None: - search_text = " ".join( - text for text in (label, display_name, description) if text is not None - ) - return cls( - sql=""" - SELECT - :type AS type, - :label AS label, - :description AS description, - :url AS url, - :search_text AS search_text, - :display_name AS display_name - """, - params={ - "type": item_type, - "label": label, - "description": description, - "url": url, - "search_text": search_text, - "display_name": display_name, - }, - ) - - -_PARAM_RE = re.compile(r"(? str: - return "/".join( - str(part) for part in (self.parent, self.child) if part is not None - ) - - def __repr__(self) -> str: - return "{}(parent={!r}, child={!r})".format( - self.__class__.__name__, self.parent, self.child - ) - - @property - def private(self) -> bool: - """ - Whether this resource is private (accessible to actor but not anonymous). - - This property is only available on Resource objects returned from - allowed_resources() when include_is_private=True is used. - - Raises: - AttributeError: If accessed without calling include_is_private=True - """ - if self._private is None: - raise AttributeError( - "The 'private' attribute is only available when using " - "allowed_resources(..., include_is_private=True)" - ) - return self._private - - @private.setter - def private(self, value: bool): - self._private = value - - @classmethod - def __init_subclass__(cls): - """ - Validate resource hierarchy doesn't exceed 2 levels. - - Raises: - ValueError: If this resource would create a 3-level hierarchy - """ - super().__init_subclass__() - - if cls.parent_class is None: - return # Top of hierarchy, nothing to validate - - # Check if our parent has a parent - that would create 3 levels - if cls.parent_class.parent_class is not None: - # We have a parent, and that parent has a parent - # This creates a 3-level hierarchy, which is not allowed - raise ValueError( - f"Resource {cls.__name__} creates a 3-level hierarchy: " - f"{cls.parent_class.parent_class.__name__} -> {cls.parent_class.__name__} -> {cls.__name__}. " - f"Maximum 2 levels allowed (parent -> child)." - ) - - @classmethod - @abstractmethod - async def resources_sql(cls, datasette, actor=None) -> str: - """ - Return SQL query that returns all resources of this type. - - Must return two columns: parent, child - """ - pass - - -class AllowedResource(NamedTuple): - """A resource with the reason it was allowed (for debugging).""" - - resource: Resource - reason: str - - -@dataclass(frozen=True, kw_only=True) -class Action: - name: str - description: str | None - abbr: str | None = None - resource_class: type[Resource] | None = None - also_requires: str | None = None # Optional action name that must also be allowed - - @property - def takes_parent(self) -> bool: - """ - Whether this action requires a parent identifier when instantiating its resource. - - Returns False for global-only actions (no resource_class). - Returns True for all actions with a resource_class (all resources require a parent identifier). - """ - return self.resource_class is not None - - @property - def takes_child(self) -> bool: - """ - Whether this action requires a child identifier when instantiating its resource. - - Returns False for global actions (no resource_class). - Returns False for parent-level resources (DatabaseResource - parent_class is None). - Returns True for child-level resources (TableResource, QueryResource - have a parent_class). - """ - if self.resource_class is None: - return False - return self.resource_class.parent_class is not None - - -_reason_id = 1 - - -@dataclass -class PermissionSQL: - """ - A plugin contributes SQL that yields: - parent TEXT NULL, - child TEXT NULL, - allow INTEGER, -- 1 allow, 0 deny - reason TEXT - - For restriction-only plugins, sql can be None and only restriction_sql is provided. - """ - - sql: str | None = ( - None # SQL that SELECTs the 4 columns above (can be None for restriction-only) - ) - params: dict[str, Any] | None = ( - None # bound params for the SQL (values only; no ':' prefix) - ) - source: str | None = None # System will set this to the plugin name - restriction_sql: str | None = ( - None # Optional SQL that returns (parent, child) for restriction filtering - ) - - @classmethod - def allow(cls, reason: str, _allow: bool = True) -> "PermissionSQL": - global _reason_id - i = _reason_id - _reason_id += 1 - return cls( - sql=f"SELECT NULL AS parent, NULL AS child, {1 if _allow else 0} AS allow, :reason_{i} AS reason", - params={f"reason_{i}": reason}, - ) - - @classmethod - def deny(cls, reason: str) -> "PermissionSQL": - return cls.allow(reason=reason, _allow=False) - - -# This is obsolete, replaced by Action and ResourceType @dataclass class Permission: name: str - abbr: str | None - description: str | None + abbr: Optional[str] + description: Optional[str] takes_database: bool takes_resource: bool default: bool diff --git a/datasette/plugins.py b/datasette/plugins.py index 5a31cdad..3769a209 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -23,14 +23,9 @@ DEFAULT_PLUGINS = ( "datasette.sql_functions", "datasette.actor_auth_cookie", "datasette.default_permissions", - "datasette.default_permissions.tokens", - "datasette.default_actions", - "datasette.default_column_types", "datasette.default_magic_parameters", "datasette.blob_renderer", - "datasette.default_debug_menu", - "datasette.default_jump_items", - "datasette.default_database_actions", + "datasette.default_menu_links", "datasette.handle_exception", "datasette.forbidden", "datasette.events", @@ -54,7 +49,7 @@ def after(outcome, hook_name, hook_impls, kwargs): results = outcome.get_result() if not isinstance(results, list): results = [results] - print("Results:", file=sys.stderr) + print(f"Results:", file=sys.stderr) pprint(results, width=40, indent=4, stream=sys.stderr) @@ -98,24 +93,21 @@ def get_plugins(): for plugin in pm.get_plugins(): static_path = None templates_path = None - plugin_name = ( - plugin.__name__ - if hasattr(plugin, "__name__") - else plugin.__class__.__name__ - ) - if plugin_name not in DEFAULT_PLUGINS: + if plugin.__name__ not in DEFAULT_PLUGINS: try: - if (importlib_resources.files(plugin_name) / "static").is_dir(): - static_path = str(importlib_resources.files(plugin_name) / "static") - if (importlib_resources.files(plugin_name) / "templates").is_dir(): + if (importlib_resources.files(plugin.__name__) / "static").is_dir(): + static_path = str( + importlib_resources.files(plugin.__name__) / "static" + ) + if (importlib_resources.files(plugin.__name__) / "templates").is_dir(): templates_path = str( - importlib_resources.files(plugin_name) / "templates" + importlib_resources.files(plugin.__name__) / "templates" ) except (TypeError, ModuleNotFoundError): # Caused by --plugins_dir= plugins pass plugin_info = { - "name": plugin_name, + "name": plugin.__name__, "static_path": static_path, "templates_path": templates_path, "hooks": [h.name for h in pm.get_hookcallers(plugin)], diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 63d22fe8..11721039 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -23,9 +23,7 @@ def publish_subcommand(publish): help="Application name to use when building", ) @click.option( - "--service", - default="", - help="Cloud Run service to deploy (or over-write)", + "--service", default="", help="Cloud Run service to deploy (or over-write)" ) @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") @click.option( @@ -303,7 +301,6 @@ def get_existing_services(): "url": service["status"]["address"]["url"], } for service in services - if "url" in service["status"] ] diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..483c81e9 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -20,7 +20,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): if column in json_cols: try: value = json.loads(value) - except (TypeError, ValueError): + except (TypeError, ValueError) as e: pass new_row.append(value) new_rows.append(new_row) diff --git a/datasette/resources.py b/datasette/resources.py deleted file mode 100644 index ee2e6d98..00000000 --- a/datasette/resources.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Core resource types for Datasette's permission system.""" - -from datasette.permissions import Resource - - -class DatabaseResource(Resource): - """A database in Datasette.""" - - name = "database" - parent_class = None # Top of the resource hierarchy - - def __init__(self, database: str): - super().__init__(parent=database, child=None) - - @classmethod - async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT database_name AS parent, NULL AS child - FROM catalog_databases - """ - - -class TableResource(Resource): - """A table in a database.""" - - name = "table" - parent_class = DatabaseResource - - def __init__(self, database: str, table: str): - super().__init__(parent=database, child=table) - - @classmethod - async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT database_name AS parent, table_name AS child - FROM catalog_tables - UNION ALL - SELECT database_name AS parent, view_name AS child - FROM catalog_views - """ - - -class QueryResource(Resource): - """A stored query in a database.""" - - name = "query" - parent_class = DatabaseResource - - def __init__(self, database: str, query: str): - super().__init__(parent=database, child=query) - - @classmethod - async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT q.database_name AS parent, q.name AS child - FROM queries q - JOIN catalog_databases cd ON cd.database_name = q.database_name - """ diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..a3117152 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -63,14 +63,6 @@ em { } /* end reset */ -/* Modal CSS variables (shared by web components via Shadow DOM) */ -:root { - --modal-backdrop-bg: rgba(0, 0, 0, 0.5); - --modal-backdrop-blur: blur(4px); - --modal-border-radius: 0.75rem; - --modal-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); - --modal-animation-duration: 0.2s; -} body { margin: 0; @@ -362,32 +354,6 @@ form.nav-menu-logout { .nav-menu-inner a { display: block; } -.nav-menu-inner button.button-as-link { - display: block; - width: 100%; - text-align: left; - font: inherit; -} -.nav-menu-inner .keyboard-shortcut { - float: right; - box-sizing: border-box; - min-width: 1.4em; - margin-left: 0.75rem; - padding: 0 0.35em; - border: 1px solid rgba(255,255,244,0.6); - border-radius: 3px; - background: rgba(255,255,244,0.12); - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.85em; - line-height: 1.35; - text-align: center; - text-decoration: none; -} -@media (max-width: 640px) { - .nav-menu-inner .keyboard-shortcut { - display: none; - } -} /* Table/database actions menu */ .page-action-menu { @@ -681,14 +647,10 @@ button.core[type=button] { border-radius: 3px; -webkit-appearance: none; padding: 9px 4px; - font-size: 16px; + font-size: 1em; font-family: Helvetica, sans-serif; } -#_search { - font-size: 16px; -} - @@ -768,474 +730,6 @@ p.zero-results { .select-wrapper.small-screen-only { display: none; } - -@keyframes datasette-modal-slide-in { - from { - opacity: 0; - transform: translateY(-20px) scale(0.95); - } - to { - opacity: 1; - transform: translateY(0) scale(1); - } -} - -@keyframes datasette-modal-fade-in { - from { opacity: 0; } - to { opacity: 1; } -} - -dialog.mobile-column-actions-dialog { - --ink: #0f0f0f; - --paper: #f5f3ef; - --muted: #6b6b6b; - --rule: #e2dfd8; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(420px, calc(100vw - 32px)); - max-width: 95vw; - max-height: min(640px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.mobile-column-actions-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.mobile-column-actions-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.mobile-column-actions-dialog .modal-header { - padding: 20px 24px 16px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.mobile-column-actions-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.mobile-column-actions-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -.mobile-column-actions-dialog .list-wrap { - flex: 1 1 auto; - min-height: 0; - overflow-y: auto; - overflow-x: hidden; - position: relative; - overscroll-behavior: contain; - -webkit-overflow-scrolling: touch; -} - -.mobile-column-actions-dialog .list-wrap::before, -.mobile-column-actions-dialog .list-wrap::after { - content: ""; - position: sticky; - display: block; - left: 0; - right: 0; - height: 20px; - pointer-events: none; - z-index: 5; -} - -.mobile-column-actions-dialog .list-wrap::before { - top: 0; - background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent); -} - -.mobile-column-actions-dialog .list-wrap::after { - bottom: 0; - background: linear-gradient(to top, rgba(255,255,255,0.9), transparent); - margin-top: -20px; -} - -.mobile-column-top-actions { - padding: 10px 24px 0; -} - -.mobile-column-top-action { - display: inline-block; - text-decoration: none; -} - -.mobile-column-section { - border-bottom: 1px solid var(--rule); -} - -.mobile-column-actions-dialog .col-header { - width: 100%; - padding: 12px 24px; - font: inherit; - font-weight: 600; - border: 0; - background: none; - cursor: pointer; - display: flex; - justify-content: space-between; - align-items: center; - text-align: left; -} - -.mobile-column-header-text { - display: flex; - flex-direction: column; - gap: 0.15rem; -} - -.mobile-column-name { - color: var(--ink); -} - -.mobile-column-meta { - color: var(--muted); - font-size: 0.78em; - font-family: ui-monospace, monospace; - font-weight: normal; -} - -.mobile-column-chevron { - color: var(--muted); - transition: transform 0.2s ease-out; -} - -.mobile-column-actions-dialog .col-header[aria-expanded="true"] .mobile-column-chevron { - transform: rotate(180deg); -} - -.mobile-column-actions-dialog .col-actions[hidden] { - display: none; -} - -.mobile-column-actions-dialog .col-actions ul, -.mobile-column-actions-dialog .col-actions li { - margin: 0; - padding: 0; - list-style-type: none; -} - -.mobile-column-actions-dialog .col-actions a, -.mobile-column-actions-dialog .col-actions button { - display: block; - width: 100%; - padding: 10px 24px 10px 40px; - color: var(--ink); - text-align: left; - font: inherit; - text-decoration: none; - background: none; - border: 0; - border-top: 1px solid #f5f5f5; - cursor: pointer; -} - -.mobile-column-actions-dialog .col-actions a:hover, -.mobile-column-actions-dialog .col-actions button:hover { - background: var(--paper); -} - -.mobile-column-actions-dialog .col-actions a:active, -.mobile-column-actions-dialog .col-actions button:active { - background: #eee; -} - -.mobile-column-description, -.mobile-column-no-actions { - margin: 0; - padding: 0 24px 12px 24px; - color: var(--muted); - font-size: 0.85em; -} - -.mobile-column-actions-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.mobile-column-actions-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.mobile-column-actions-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.mobile-column-actions-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.mobile-column-actions-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -dialog.set-column-type-dialog { - --ink: #0f0f0f; - --paper: #f5f3ef; - --muted: #6b6b6b; - --rule: #e2dfd8; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; - max-height: min(720px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.set-column-type-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.set-column-type-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.set-column-type-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.set-column-type-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.set-column-type-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -.set-column-type-status, -.set-column-type-empty, -.set-column-type-error { - margin: 0; - padding: 12px 24px 0; -} - -.set-column-type-status, -.set-column-type-empty { - color: var(--muted); - font-size: 0.9rem; -} - -.set-column-type-error { - color: #b91c1c; - font-size: 0.9rem; -} - -.set-column-type-options { - padding: 16px 24px 24px; - overflow-y: auto; - display: grid; - gap: 12px; -} - -.set-column-type-option { - display: grid; - grid-template-columns: auto 1fr; - gap: 12px; - align-items: start; - padding: 14px 16px; - border: 1px solid var(--rule); - border-radius: 8px; - background: #fcfbf9; - cursor: pointer; -} - -.set-column-type-option:focus-within { - border-color: var(--accent); - box-shadow: 0 0 0 3px rgba(26, 86, 219, 0.12); -} - -.set-column-type-option input { - margin-top: 3px; -} - -.set-column-type-option-content { - display: grid; - gap: 4px; -} - -.set-column-type-option-name { - font-family: ui-monospace, monospace; - font-size: 0.95rem; - color: var(--ink); -} - -.set-column-type-option-description { - color: var(--muted); - font-size: 0.9rem; -} - -.set-column-type-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.set-column-type-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.set-column-type-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.set-column-type-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.set-column-type-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.set-column-type-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.set-column-type-dialog .btn-primary:hover { - background: #1949b8; -} - -.set-column-type-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - -@media (max-width: 640px) { - dialog.mobile-column-actions-dialog { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; - } - - .mobile-column-actions-dialog .modal-header { - padding: 16px 18px 14px; - } - - .mobile-column-top-actions { - padding-left: 18px; - padding-right: 18px; - } - - .mobile-column-actions-dialog .col-header { - padding-left: 18px; - padding-right: 18px; - } - - .mobile-column-actions-dialog .col-actions a, - .mobile-column-actions-dialog .col-actions button { - padding-left: 34px; - padding-right: 18px; - } - - .mobile-column-description, - .mobile-column-no-actions { - padding-left: 18px; - padding-right: 18px; - } - - dialog.set-column-type-dialog { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; - } - - .set-column-type-dialog .modal-header, - .set-column-type-status, - .set-column-type-empty, - .set-column-type-error, - .set-column-type-options { - padding-left: 18px; - padding-right: 18px; - } -} - @media only screen and (max-width: 576px) { .small-screen-only { @@ -1297,43 +791,6 @@ dialog.set-column-type-dialog::backdrop { .filters input.filter-value { width: 140px; } - button.choose-columns-mobile, - button.column-actions-mobile { - display: inline-flex; - align-items: center; - justify-content: center; - padding: 0.5rem 1rem; - margin-bottom: 1em; - font-size: 0.9rem; - line-height: 1.2; - font-family: inherit; - background: white; - border: 1px solid #ccc; - border-radius: 5px; - cursor: pointer; - vertical-align: top; - box-sizing: border-box; - min-height: 2.5rem; - } - - button.column-actions-mobile { - gap: 0.55rem; - } - - button.column-actions-mobile svg { - display: block; - width: 16px; - height: 16px; - flex-shrink: 0; - } - - button.column-actions-mobile span { - line-height: 1.2; - } - - button.choose-columns-mobile { - margin-right: 0.5rem; - } } svg.dropdown-menu-icon { @@ -1409,15 +866,11 @@ svg.dropdown-menu-icon { border-bottom: 5px solid #666; } -.stored-query-edit-sql { +.canned-query-edit-sql { padding-left: 0.5em; position: relative; top: 1px; } -.save-query { - display: inline-block; - margin-left: 0.45em; -} .blob-download { display: block; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js deleted file mode 100644 index 133e7cb0..00000000 --- a/datasette/static/column-chooser.js +++ /dev/null @@ -1,699 +0,0 @@ -class ColumnChooser extends HTMLElement { - constructor() { - super(); - this.attachShadow({ mode: "open" }); - - // State - this._items = []; - this._checked = new Set(); - this._savedItems = null; - this._savedChecked = null; - this._onApply = null; - - // Drag state - this._ghost = null; - this._dragSrcIdx = null; - this._dropTargetIdx = null; - this._dropPosition = null; - this._ghostOffX = 0; - this._ghostOffY = 0; - this._autoScrollRAF = null; - this._lastPointerY = 0; - this._lastPointerX = 0; - this._SCROLL_ZONE = 72; - this._SCROLL_SPEED = 0.4; - - // Bound handlers - this._onMove = this._onMove.bind(this); - this._onUp = this._onUp.bind(this); - - this.shadowRoot.innerHTML = ` - - - -
- Choose columns - -
-
- - -
-
-
-
-
    -
    -
    - - - -
    -     - `; - - // DOM refs - this._dialog = this.shadowRoot.querySelector("dialog"); - this._listWrap = this.shadowRoot.getElementById("listWrap"); - this._dragList = this.shadowRoot.getElementById("dragList"); - this._pulseTop = this.shadowRoot.getElementById("pulseTop"); - this._pulseBot = this.shadowRoot.getElementById("pulseBot"); - this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); - this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); - this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); - this._applyBtn = this.shadowRoot.getElementById("applyBtn"); - this._countEl = this.shadowRoot.getElementById("selectedCount"); - this._footerEl = this.shadowRoot.getElementById("footerInfo"); - - // Event listeners - this._selectAllBtn.addEventListener("click", () => this._selectAll()); - this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); - this._cancelBtn.addEventListener("click", () => this._close()); - this._applyBtn.addEventListener("click", () => this._apply()); - this._dialog.addEventListener("click", (e) => { - if (e.target === this._dialog) this._close(); - }); - this._dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this._close(); - }); - } - - /** - * Open the column chooser dialog. - * @param {Object} opts - * @param {string[]} opts.columns - All available column names, in display order. - * @param {string[]} opts.selected - Column names that should be pre-checked. - * @param {function(string[]): void} opts.onApply - Called with the selected columns in order when Apply is clicked. - */ - open({ columns, selected = [], onApply }) { - this._items = [...columns]; - this._checked = new Set(selected); - this._onApply = onApply || null; - - // Save state for cancel/restore - this._savedItems = [...this._items]; - this._savedChecked = new Set(this._checked); - - this._render(); - this._dialog.showModal(); - } - - // ── Internal methods ── - - _close() { - this._items = this._savedItems ? [...this._savedItems] : this._items; - this._checked = this._savedChecked - ? new Set(this._savedChecked) - : this._checked; - this._dialog.close(); - } - - _selectAll() { - this._items.forEach((col) => this._checked.add(col)); - this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { - cb.checked = true; - }); - this._updateCounts(); - } - - _deselectAll() { - this._checked.clear(); - this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { - cb.checked = false; - }); - this._updateCounts(); - } - - _apply() { - const selected = this._items.filter((col) => this._checked.has(col)); - this._dialog.close(); - if (this._onApply) { - this._onApply(selected); - } - } - - _render() { - this._dragList.innerHTML = ""; - this._items.forEach((col, i) => { - const li = document.createElement("li"); - li.className = "drag-item"; - li.dataset.idx = i; - li.innerHTML = ` - - - - - - - - - - - -
    - `; - - li.querySelector("input").addEventListener("change", (e) => { - e.target.checked ? this._checked.add(col) : this._checked.delete(col); - this._updateCounts(); - }); - - li.querySelector(".drag-handle").addEventListener("pointerdown", (e) => - this._startDrag(e, i), - ); - this._dragList.appendChild(li); - }); - - this._updateCounts(); - } - - _updateCounts() { - const n = this._checked.size; - this._countEl.textContent = `${n} of ${this._items.length} selected`; - this._footerEl.textContent = `${this._items.length} columns`; - } - - // ── Drag engine ── - - _startDrag(e, idx) { - e.preventDefault(); - this._dragSrcIdx = idx; - - const srcEl = this._dragList.children[idx]; - const rect = srcEl.getBoundingClientRect(); - - this._ghostOffX = e.clientX - rect.left; - this._ghostOffY = e.clientY - rect.top; - - // Build ghost inside shadow DOM - this._ghost = document.createElement("div"); - this._ghost.className = "drag-ghost"; - this._ghost.style.width = rect.width + "px"; - this._ghost.style.height = rect.height + "px"; - this._ghost.innerHTML = srcEl.innerHTML; - this._ghost.querySelector(".drop-indicator")?.remove(); - const h = this._ghost.querySelector(".drag-handle"); - if (h) h.style.color = "var(--accent)"; - this.shadowRoot.appendChild(this._ghost); - - srcEl.classList.add("is-dragging"); - this._positionGhost(e.clientX, e.clientY); - - document.addEventListener("pointermove", this._onMove); - document.addEventListener("pointerup", this._onUp); - document.addEventListener("pointercancel", this._onUp); - } - - _positionGhost(cx, cy) { - this._ghost.style.left = cx - this._ghostOffX + "px"; - this._ghost.style.top = cy - this._ghostOffY + "px"; - } - - _onMove(e) { - this._lastPointerX = e.clientX; - this._lastPointerY = e.clientY; - this._positionGhost(e.clientX, e.clientY); - this._updateDropTarget(e.clientY); - this._updateAutoScroll(e.clientY); - } - - _onUp() { - document.removeEventListener("pointermove", this._onMove); - document.removeEventListener("pointerup", this._onUp); - document.removeEventListener("pointercancel", this._onUp); - - this._stopAutoScroll(); - - const noMove = - this._dropTargetIdx === null || this._dropTargetIdx === this._dragSrcIdx; - this._clearDropIndicators(); - - let dest = null; - if (!noMove) { - const moved = this._items.splice(this._dragSrcIdx, 1)[0]; - dest = this._dropTargetIdx; - if (this._dropPosition === "after") dest++; - if (dest > this._dragSrcIdx) dest--; - this._items.splice(dest, 0, moved); - } - - this._dragSrcIdx = null; - this._dropTargetIdx = null; - this._dropPosition = null; - - const g = this._ghost; - this._ghost = null; - - if (noMove) { - if (g) g.remove(); - this._render(); - return; - } - - this._render(); - - if (g && dest !== null) { - const landedEl = this._dragList.children[dest]; - if (landedEl) { - landedEl.style.opacity = "0"; - const r = landedEl.getBoundingClientRect(); - g.getBoundingClientRect(); - g.style.transition = - "left 0.15s cubic-bezier(0.22, 1, 0.36, 1), top 0.15s cubic-bezier(0.22, 1, 0.36, 1), box-shadow 0.15s, opacity 0.1s 0.1s"; - g.style.left = r.left + "px"; - g.style.top = r.top + "px"; - g.style.boxShadow = "0 1px 4px rgba(0,0,0,0.08)"; - g.style.opacity = "0"; - setTimeout(() => { - g.remove(); - if (landedEl) landedEl.style.opacity = ""; - }, 160); - } else { - g.remove(); - } - } else if (g) { - g.remove(); - } - } - - _updateDropTarget(clientY) { - this._clearDropIndicators(); - const listItems = [ - ...this._dragList.querySelectorAll(".drag-item:not(.is-dragging)"), - ]; - if (!listItems.length) return; - - let best = null, - bestDist = Infinity; - listItems.forEach((li) => { - const r = li.getBoundingClientRect(); - const mid = r.top + r.height / 2; - const dist = Math.abs(clientY - mid); - if (dist < bestDist) { - bestDist = dist; - best = li; - } - }); - - if (!best) return; - const r = best.getBoundingClientRect(); - const mid = r.top + r.height / 2; - const above = clientY < mid; - const indic = best.querySelector(".drop-indicator"); - - this._dropTargetIdx = parseInt(best.dataset.idx); - this._dropPosition = above ? "before" : "after"; - - if (indic) { - indic.className = "drop-indicator " + (above ? "top" : "bottom"); - } - } - - _clearDropIndicators() { - this._dragList.querySelectorAll(".drop-indicator").forEach((el) => { - el.className = "drop-indicator"; - }); - } - - _updateAutoScroll(clientY) { - const rect = this._listWrap.getBoundingClientRect(); - const relY = clientY - rect.top; - const distTop = relY; - const distBot = rect.height - relY; - - const inTop = distTop < this._SCROLL_ZONE && distTop >= 0; - const inBot = distBot < this._SCROLL_ZONE && distBot >= 0; - - this._pulseTop.classList.toggle("active", inTop); - this._pulseBot.classList.toggle("active", inBot); - - if ((inTop || inBot) && !this._autoScrollRAF) { - let lastTime = null; - const loop = (ts) => { - if (!this._ghost) { - this._stopAutoScroll(); - return; - } - if (lastTime !== null) { - const dt = ts - lastTime; - const rect2 = this._listWrap.getBoundingClientRect(); - const relY2 = this._lastPointerY - rect2.top; - const dTop = relY2; - const dBot = rect2.height - relY2; - - if (dTop < this._SCROLL_ZONE && dTop >= 0) { - const factor = 1 - dTop / this._SCROLL_ZONE; - this._listWrap.scrollTop -= this._SCROLL_SPEED * dt * factor * 2.5; - } else if (dBot < this._SCROLL_ZONE && dBot >= 0) { - const factor = 1 - dBot / this._SCROLL_ZONE; - this._listWrap.scrollTop += this._SCROLL_SPEED * dt * factor * 2.5; - } else { - this._stopAutoScroll(); - return; - } - this._updateDropTarget(this._lastPointerY); - } - lastTime = ts; - this._autoScrollRAF = requestAnimationFrame(loop); - }; - this._autoScrollRAF = requestAnimationFrame(loop); - } - - if (!inTop && !inBot) this._stopAutoScroll(); - } - - _stopAutoScroll() { - if (this._autoScrollRAF) { - cancelAnimationFrame(this._autoScrollRAF); - this._autoScrollRAF = null; - } - this._pulseTop.classList.remove("active"); - this._pulseBot.classList.remove("active"); - } -} - -customElements.define("column-chooser", ColumnChooser); diff --git a/datasette/static/datasette-manager.js b/datasette/static/datasette-manager.js index e75f7aae..10716cc5 100644 --- a/datasette/static/datasette-manager.js +++ b/datasette/static/datasette-manager.js @@ -82,19 +82,6 @@ const datasetteManager = { return columnActions; }, - makeJumpSections: (context) => { - let jumpSections = []; - - datasetteManager.plugins.forEach((plugin) => { - if (plugin.makeJumpSections) { - const sections = plugin.makeJumpSections(context) || []; - jumpSections.push(...sections); - } - }); - - return jumpSections; - }, - /** * In MVP, each plugin can only have 1 instance. * In future, panels could be repeated. We omit that for now since so many plugins depend on @@ -106,12 +93,12 @@ const datasetteManager = { */ renderAboveTablePanel: () => { const aboveTablePanel = document.querySelector( - DOM_SELECTORS.aboveTablePanel, + DOM_SELECTORS.aboveTablePanel ); if (!aboveTablePanel) { console.warn( - "This page does not have a table, the renderAboveTablePanel cannot be used.", + "This page does not have a table, the renderAboveTablePanel cannot be used." ); return; } @@ -205,6 +192,7 @@ const initializeDatasette = () => { // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window. window.__DATASETTE__ = datasetteManager; + console.debug("Datasette Manager Created!"); const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, { detail: datasetteManager, diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js index 0e6e2c29..d83b8186 100644 --- a/datasette/static/json-format-highlight-1.0.1.js +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -7,8 +7,8 @@ MIT Licensed typeof exports === "object" && typeof module !== "undefined" ? (module.exports = factory()) : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); + ? define(factory) + : (global.jsonFormatHighlight = factory()); })(this, function () { "use strict"; @@ -42,13 +42,13 @@ MIT Licensed color = /true/.test(match) ? colors.trueColor : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; } return '' + match + ""; - }, + } ); } diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js deleted file mode 100644 index a386b1fc..00000000 --- a/datasette/static/mobile-column-actions.js +++ /dev/null @@ -1,318 +0,0 @@ -var MOBILE_COLUMN_BREAKPOINT = 576; -var MOBILE_COLUMN_DIALOG_ID = "mobile-column-actions-dialog"; -var MOBILE_COLUMN_DIALOG_TITLE_ID = "mobile-column-actions-title"; - -function mobileColumnHeaders(manager) { - return Array.from( - document.querySelectorAll(manager.selectors.tableHeaders), - ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1"); -} - -function mobileColumnMetaText(th) { - var parts = []; - if (th.dataset.columnType) { - parts.push(th.dataset.columnType); - } - if (th.dataset.isPk === "1") { - parts.push("pk"); - } - if (th.dataset.columnNotNull === "1") { - parts.push("not null"); - } - return parts.join(", "); -} - -function createMobileColumnActionNode(itemConfig, closeDialog) { - var actionNode; - if (itemConfig.href) { - actionNode = document.createElement("a"); - actionNode.href = itemConfig.href; - } else { - actionNode = document.createElement("button"); - actionNode.type = "button"; - } - actionNode.textContent = itemConfig.label; - - if (itemConfig.onClick) { - actionNode.addEventListener("click", function (ev) { - try { - itemConfig.onClick.call(actionNode, ev); - } finally { - closeDialog({ restoreFocus: false }); - } - }); - } - - return actionNode; -} - -function initMobileColumnActions(manager) { - var triggerButton = document.querySelector(".column-actions-mobile"); - if (!triggerButton) { - return; - } - - if ( - !window.URLSearchParams || - !window.HTMLDialogElement || - !manager.columnActions - ) { - triggerButton.style.display = "none"; - return; - } - - if (!mobileColumnHeaders(manager).length) { - triggerButton.style.display = "none"; - return; - } - - var dialog = document.createElement("dialog"); - dialog.className = "mobile-column-actions-dialog"; - dialog.id = MOBILE_COLUMN_DIALOG_ID; - dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); - dialog.innerHTML = ` -
    - Column actions - -
    -
    -
    - Tap a column to reveal actions. - -
    - `; - document.body.appendChild(dialog); - - triggerButton.setAttribute("aria-haspopup", "dialog"); - triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); - triggerButton.setAttribute("aria-expanded", "false"); - - var countEl = dialog.querySelector(".modal-meta"); - var listWrap = dialog.querySelector(".mobile-column-list"); - var doneButton = dialog.querySelector(".mobile-column-actions-done"); - var expandedSectionId = null; - var shouldRestoreFocus = true; - - function updateExpandedSection() { - Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { - var controlsId = button.getAttribute("aria-controls"); - var actionList = dialog.querySelector("#" + controlsId); - var isExpanded = controlsId === expandedSectionId; - button.setAttribute("aria-expanded", isExpanded ? "true" : "false"); - actionList.hidden = !isExpanded; - actionList.classList.toggle("expanded", isExpanded); - }); - } - - function scrollExpandedSectionIntoView(section) { - var sectionTop = section.offsetTop; - var sectionBottom = sectionTop + section.offsetHeight; - var visibleTop = listWrap.scrollTop; - var visibleBottom = visibleTop + listWrap.clientHeight; - var sectionHeight = section.offsetHeight; - - if (sectionTop < visibleTop) { - listWrap.scrollTop = sectionTop; - return; - } - - if (sectionBottom <= visibleBottom) { - return; - } - - if (sectionHeight <= listWrap.clientHeight) { - listWrap.scrollTop = sectionBottom - listWrap.clientHeight; - } else { - listWrap.scrollTop = sectionTop; - } - } - - function closeDialog(options) { - options = options || {}; - shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } - } - } - - function renderDialog() { - var headers = mobileColumnHeaders(manager); - if (!headers.length) { - closeDialog({ restoreFocus: false }); - triggerButton.style.display = "none"; - return false; - } - - if ( - !headers.some( - (_th, index) => `mobile-column-actions-${index}` === expandedSectionId, - ) - ) { - expandedSectionId = null; - } - - countEl.textContent = `${headers.length} column${ - headers.length === 1 ? "" : "s" - }`; - listWrap.innerHTML = ""; - - if (manager.columnActions.shouldShowShowAllColumns()) { - var topActions = document.createElement("div"); - topActions.className = "mobile-column-top-actions"; - - var showAllColumns = document.createElement("a"); - showAllColumns.className = "btn btn-ghost mobile-column-top-action"; - showAllColumns.href = manager.columnActions.showAllColumnsUrl(); - showAllColumns.textContent = "Show all columns"; - - topActions.appendChild(showAllColumns); - listWrap.appendChild(topActions); - } - - headers.forEach((th, index) => { - var sectionId = `mobile-column-actions-${index}`; - var actionState = manager.columnActions.buildColumnActionState(th, { - includeChooseColumns: false, - includeShowAllColumns: false, - }); - var section = document.createElement("section"); - section.className = "mobile-column-section"; - - var headerButton = document.createElement("button"); - headerButton.type = "button"; - headerButton.className = "col-header"; - headerButton.setAttribute("aria-controls", sectionId); - headerButton.setAttribute("aria-expanded", "false"); - - var headerText = document.createElement("span"); - headerText.className = "mobile-column-header-text"; - - var name = document.createElement("span"); - name.className = "mobile-column-name"; - name.textContent = th.dataset.column; - headerText.appendChild(name); - - var metaText = mobileColumnMetaText(th); - if (metaText) { - var meta = document.createElement("span"); - meta.className = "mobile-column-meta"; - meta.textContent = metaText; - headerText.appendChild(meta); - } - - var chevron = document.createElement("span"); - chevron.className = "mobile-column-chevron"; - chevron.setAttribute("aria-hidden", "true"); - chevron.textContent = "▾"; - - headerButton.appendChild(headerText); - headerButton.appendChild(chevron); - headerButton.addEventListener("click", function () { - expandedSectionId = expandedSectionId === sectionId ? null : sectionId; - updateExpandedSection(); - if (expandedSectionId === sectionId) { - scrollExpandedSectionIntoView(section); - } - }); - - var actionContainer = document.createElement("div"); - actionContainer.id = sectionId; - actionContainer.className = "col-actions"; - actionContainer.hidden = true; - - if (actionState.columnDescription) { - var description = document.createElement("p"); - description.className = "mobile-column-description"; - description.textContent = actionState.columnDescription; - actionContainer.appendChild(description); - } - - if (actionState.actionItems.length) { - var actionList = document.createElement("ul"); - actionState.actionItems.forEach((itemConfig) => { - var actionItem = document.createElement("li"); - actionItem.appendChild( - createMobileColumnActionNode(itemConfig, closeDialog), - ); - actionList.appendChild(actionItem); - }); - actionContainer.appendChild(actionList); - } else { - var noActions = document.createElement("p"); - noActions.className = "mobile-column-no-actions"; - noActions.textContent = "No actions available"; - actionContainer.appendChild(noActions); - } - - section.appendChild(headerButton); - section.appendChild(actionContainer); - listWrap.appendChild(section); - }); - - updateExpandedSection(); - return true; - } - - function openDialog() { - if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT) { - return; - } - if (!renderDialog()) { - return; - } - if (!dialog.open) { - dialog.showModal(); - } - triggerButton.setAttribute("aria-expanded", "true"); - var focusTarget = - dialog.querySelector(".mobile-column-top-action") || - dialog.querySelector(".col-header") || - doneButton; - focusTarget.focus(); - } - - triggerButton.addEventListener("click", function () { - if (dialog.open) { - closeDialog(); - } else { - openDialog(); - } - }); - - doneButton.addEventListener("click", function () { - closeDialog(); - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeDialog(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeDialog(); - }); - - dialog.addEventListener("close", function () { - triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } - }); - - window.addEventListener("resize", function () { - if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT && dialog.open) { - closeDialog({ restoreFocus: false }); - } - }); -} - -document.addEventListener("datasette_init", function (evt) { - initMobileColumnActions(evt.detail); -}); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js deleted file mode 100644 index ec2d23d8..00000000 --- a/datasette/static/navigation-search.js +++ /dev/null @@ -1,910 +0,0 @@ -let navigationSearchInstanceCounter = 0; - -class NavigationSearch extends HTMLElement { - constructor() { - super(); - this.instanceId = ++navigationSearchInstanceCounter; - this.inputId = `navigation-search-input-${this.instanceId}`; - this.instructionsId = `navigation-search-instructions-${this.instanceId}`; - this.listboxId = `navigation-search-results-${this.instanceId}`; - this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; - this.statusId = `navigation-search-status-${this.instanceId}`; - this.titleId = `navigation-search-title-${this.instanceId}`; - this.attachShadow({ mode: "open" }); - this.selectedIndex = -1; - this.matches = []; - this.renderedMatches = []; - this.debounceTimer = null; - this.restoreFocusTarget = null; - this.shouldRestoreFocus = true; - - this.render(); - this.setupEventListeners(); - } - - render() { - this.shadowRoot.innerHTML = ` - - - -
    - - - -
    - - -
    -
    -
    - ↑ ↓ Navigate - Enter Select - Esc Close -
    -
    -     - `; - } - - setupEventListeners() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const closeButton = this.shadowRoot.querySelector(".close-search"); - const resultsContainer = - this.shadowRoot.querySelector(".results-container"); - - // Global keyboard listener for "/" - document.addEventListener("keydown", (e) => { - if (e.key === "/" && !this.isInputFocused() && !dialog.open) { - e.preventDefault(); - this.openMenu(); - } - }); - - document.addEventListener("click", (e) => { - const trigger = e.target.closest("[data-navigation-search-open]"); - if (trigger) { - e.preventDefault(); - const details = trigger.closest("details"); - const restoreTarget = details?.querySelector("summary") || trigger; - details?.removeAttribute("open"); - this.openMenu(restoreTarget); - } - }); - - // Input event - input.addEventListener("input", (e) => { - this.handleSearch(e.target.value); - }); - - // Keyboard navigation - input.addEventListener("keydown", (e) => { - if (e.key === "ArrowDown") { - e.preventDefault(); - this.moveSelection(1); - } else if (e.key === "ArrowUp") { - e.preventDefault(); - this.moveSelection(-1); - } else if (e.key === "Enter") { - e.preventDefault(); - this.selectCurrentItem(); - } else if (e.key === "Escape") { - this.closeMenu(); - } - }); - - closeButton.addEventListener("click", () => { - this.closeMenu(); - }); - - // Click on result item - resultsContainer.addEventListener("click", (e) => { - const clearRecent = e.target.closest("[data-clear-recent-items]"); - if (clearRecent) { - e.preventDefault(); - this.clearRecentItems(); - return; - } - - const item = e.target.closest(".result-item"); - if (item) { - const index = parseInt(item.dataset.index); - this.selectItem(index); - } - }); - - // Close on backdrop click - dialog.addEventListener("click", (e) => { - if (e.target === dialog) { - this.closeMenu(); - } - }); - - dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this.closeMenu(); - }); - - dialog.addEventListener("close", () => { - this.onMenuClosed(); - }); - - // Initial load - this.loadInitialData(); - } - - isInputFocused() { - const activeElement = document.activeElement; - return ( - activeElement && - (activeElement.tagName === "INPUT" || - activeElement.tagName === "TEXTAREA" || - activeElement.isContentEditable) - ); - } - - setElementAttribute(element, name, value) { - if (!element) { - return; - } - if (typeof element.setAttribute === "function") { - element.setAttribute(name, value); - } else { - element[name] = String(value); - } - } - - removeElementAttribute(element, name) { - if (!element) { - return; - } - if (typeof element.removeAttribute === "function") { - element.removeAttribute(name); - } else { - delete element[name]; - } - } - - focusRestoreTarget(trigger) { - if (trigger && typeof trigger.focus === "function") { - return trigger; - } - if ( - document.activeElement && - typeof document.activeElement.focus === "function" - ) { - return document.activeElement; - } - return null; - } - - setNavigationTriggersExpanded(expanded) { - if (typeof document.querySelectorAll !== "function") { - return; - } - document - .querySelectorAll("[data-navigation-search-open]") - .forEach((trigger) => { - this.setElementAttribute( - trigger, - "aria-expanded", - expanded ? "true" : "false", - ); - }); - } - - resultOptionId(index) { - return `${this.listboxId}-option-${index}`; - } - - updateComboboxState() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const matches = this.renderedMatches || []; - this.setElementAttribute( - input, - "aria-expanded", - dialog && dialog.open && matches.length > 0 ? "true" : "false", - ); - - if ( - dialog && - dialog.open && - this.selectedIndex >= 0 && - this.selectedIndex < matches.length - ) { - this.setElementAttribute( - input, - "aria-activedescendant", - this.resultOptionId(this.selectedIndex), - ); - } else { - this.removeElementAttribute(input, "aria-activedescendant"); - } - } - - setStatus(message) { - const status = this.shadowRoot.querySelector(`#${this.statusId}`); - if (status) { - status.textContent = message || ""; - } - } - - resultsStatus(count, truncated) { - if (truncated) { - return "More than 100 results. Keep typing to narrow the list."; - } - if (count === 0) { - return "No results found."; - } - if (count === 1) { - return "1 result."; - } - return `${count} results.`; - } - - loadInitialData() { - const itemsAttr = this.getAttribute("items"); - if (itemsAttr) { - try { - this.allItems = JSON.parse(itemsAttr); - this.matches = this.allItems; - } catch (e) { - console.error("Failed to parse items attribute:", e); - this.allItems = []; - this.matches = []; - } - } - } - - handleSearch(query) { - clearTimeout(this.debounceTimer); - if (query.trim()) { - this.setStatus("Searching..."); - } else { - this.setStatus(""); - } - - this.debounceTimer = setTimeout(() => { - const url = this.getAttribute("url"); - - if (url) { - // Fetch from API - this.fetchResults(url, query); - } else { - // Filter local items - this.filterLocalItems(query); - } - }, 200); - } - - async fetchResults(url, query) { - try { - const searchUrl = `${url}?q=${encodeURIComponent(query)}`; - const response = await fetch(searchUrl); - const data = await response.json(); - this.matches = data.matches || []; - this.selectedIndex = this.matches.length > 0 ? 0 : -1; - this.renderResults(); - if (query.trim()) { - this.setStatus(this.resultsStatus(this.matches.length, data.truncated)); - } else { - this.setStatus(""); - } - } catch (e) { - console.error("Failed to fetch search results:", e); - this.matches = []; - this.renderResults(); - this.setStatus("Search failed."); - } - } - - filterLocalItems(query) { - if (!query.trim()) { - this.matches = this.allItems || []; - } else { - const lowerQuery = query.toLowerCase(); - this.matches = (this.allItems || []).filter( - (item) => - item.name.toLowerCase().includes(lowerQuery) || - (item.display_name || "").toLowerCase().includes(lowerQuery) || - item.url.toLowerCase().includes(lowerQuery), - ); - } - this.selectedIndex = this.matches.length > 0 ? 0 : -1; - this.renderResults(); - if (query.trim()) { - this.setStatus(this.resultsStatus(this.matches.length, false)); - } else { - this.setStatus(""); - } - } - - recentItemsStorageKey() { - return "datasette.navigationSearch.recentItems"; - } - - loadRecentItems() { - if (typeof localStorage === "undefined") { - return []; - } - - try { - const raw = localStorage.getItem(this.recentItemsStorageKey()); - if (!raw) { - return []; - } - const parsed = JSON.parse(raw); - if (!Array.isArray(parsed)) { - return []; - } - return parsed - .filter((item) => item && item.name && item.url) - .map((item) => ({ - name: String(item.name), - display_name: item.display_name ? String(item.display_name) : "", - url: String(item.url), - type: item.type ? String(item.type) : "", - description: item.description ? String(item.description) : "", - })) - .slice(0, 5); - } catch (e) { - return []; - } - } - - saveRecentItem(match) { - if ( - typeof localStorage === "undefined" || - !match || - !match.name || - !match.url - ) { - return; - } - - try { - const item = { - name: String(match.name), - display_name: match.display_name ? String(match.display_name) : "", - url: String(match.url), - type: match.type ? String(match.type) : "", - description: match.description ? String(match.description) : "", - }; - const recentItems = this.loadRecentItems().filter( - (recentItem) => recentItem.url !== item.url, - ); - localStorage.setItem( - this.recentItemsStorageKey(), - JSON.stringify([item, ...recentItems].slice(0, 5)), - ); - } catch (e) { - // localStorage may be unavailable, full, or disabled. - } - } - - clearRecentItems() { - if (typeof localStorage === "undefined") { - return; - } - - try { - localStorage.removeItem(this.recentItemsStorageKey()); - } catch (e) { - localStorage.setItem(this.recentItemsStorageKey(), "[]"); - } - this.renderResults(); - this.setStatus("Recent items cleared."); - } - - jumpSections() { - const manager = window.__DATASETTE__; - if (!manager || typeof manager.makeJumpSections !== "function") { - return []; - } - const sections = manager.makeJumpSections({ - navigationSearch: this, - }); - return Array.isArray(sections) - ? sections.filter( - (section) => section && typeof section.render === "function", - ) - : []; - } - - jumpSectionsHtml(jumpSections) { - return jumpSections - .map((section, index) => { - const id = section.id - ? ` data-jump-section-id="${this.escapeHtml(section.id)}"` - : ""; - return `
    `; - }) - .join(""); - } - - renderJumpSections(container, jumpSections) { - jumpSections.forEach((section, index) => { - const node = container.querySelector( - `[data-jump-section-index="${index}"]`, - ); - if (!node) { - return; - } - section.render(node, { - navigationSearch: this, - container, - input: this.shadowRoot.querySelector(".search-input"), - }); - }); - } - - resultItemHtml(match, index) { - const displayName = match.display_name || match.name; - const label = - match.display_name && match.display_name !== match.name - ? `
    ${this.escapeHtml(match.name)}
    ` - : ""; - const type = match.type - ? `
    ${this.escapeHtml(match.type)}
    ` - : ""; - const description = match.description - ? `
    ${this.escapeHtml( - match.description, - )}
    ` - : ""; - return ` -
    -
    - ${type} -
    ${this.escapeHtml(displayName)}
    - ${label} -
    ${this.escapeHtml(match.url)}
    - ${description} -
    -
    - `; - } - - renderResults() { - const container = this.shadowRoot.querySelector(".results-container"); - const input = this.shadowRoot.querySelector(".search-input"); - const showStartContent = !input.value.trim(); - const jumpSections = showStartContent ? this.jumpSections() : []; - const startBlock = showStartContent - ? this.jumpSectionsHtml(jumpSections) - : ""; - const recentItems = showStartContent ? this.loadRecentItems() : []; - const defaultMatches = showStartContent ? [] : this.matches; - const renderedMatches = [...recentItems, ...defaultMatches]; - this.renderedMatches = renderedMatches; - const emptyListbox = `
    `; - - if (renderedMatches.length) { - if ( - this.selectedIndex < 0 || - this.selectedIndex >= renderedMatches.length - ) { - this.selectedIndex = 0; - } - } else { - this.selectedIndex = -1; - } - - if (renderedMatches.length === 0) { - if (startBlock) { - container.innerHTML = startBlock + emptyListbox; - this.renderJumpSections(container, jumpSections); - } else if (showStartContent) { - container.innerHTML = emptyListbox; - } else { - const message = input.value.trim() - ? "No results found" - : "Start typing to search..."; - container.innerHTML = `${emptyListbox}
    ${message}
    `; - } - this.updateComboboxState(); - return; - } - - const recentHeading = recentItems.length - ? `
    Recent
    ` - : ""; - const recentGroup = recentItems.length - ? `
    ${recentItems - .map((match, index) => this.resultItemHtml(match, index)) - .join("")}
    ` - : ""; - const recentActions = recentItems.length - ? `
    ` - : ""; - const defaultHtml = defaultMatches - .map((match, index) => - this.resultItemHtml(match, recentItems.length + index), - ) - .join(""); - container.innerHTML = - startBlock + - recentHeading + - `
    ${recentGroup}${defaultHtml}
    ` + - recentActions; - this.renderJumpSections(container, jumpSections); - this.updateComboboxState(); - - // Scroll selected item into view - if (this.selectedIndex >= 0) { - const selectedItem = container.querySelector( - `.result-item[data-index="${this.selectedIndex}"]`, - ); - if (selectedItem) { - selectedItem.scrollIntoView({ block: "nearest" }); - } - } - } - - moveSelection(direction) { - const matches = this.renderedMatches || this.matches; - const newIndex = this.selectedIndex + direction; - if (newIndex >= 0 && newIndex < matches.length) { - this.selectedIndex = newIndex; - this.renderResults(); - } - } - - selectCurrentItem() { - const matches = this.renderedMatches || this.matches; - if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) { - this.selectItem(this.selectedIndex); - } - } - - selectItem(index) { - const matches = this.renderedMatches || this.matches; - const match = matches[index]; - if (match) { - this.saveRecentItem(match); - - // Dispatch custom event - this.dispatchEvent( - new CustomEvent("select", { - detail: match, - bubbles: true, - composed: true, - }), - ); - - // Navigate to URL - window.location.href = match.url; - - this.closeMenu({ restoreFocus: false }); - } - } - - openMenu(trigger) { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - - this.restoreFocusTarget = this.focusRestoreTarget(trigger); - this.shouldRestoreFocus = true; - if (!dialog.open) { - dialog.showModal(); - } - this.setNavigationTriggersExpanded(true); - input.value = ""; - input.focus(); - - // Reset state, then populate the default jump list. - this.matches = []; - this.selectedIndex = -1; - this.renderResults(); - this.setStatus(""); - } - - closeMenu(options = {}) { - const dialog = this.shadowRoot.querySelector("dialog"); - this.shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - this.onMenuClosed(); - } - } - - onMenuClosed() { - const input = this.shadowRoot.querySelector(".search-input"); - this.setElementAttribute(input, "aria-expanded", "false"); - this.removeElementAttribute(input, "aria-activedescendant"); - this.setNavigationTriggersExpanded(false); - this.setStatus(""); - if ( - this.shouldRestoreFocus && - this.restoreFocusTarget && - typeof this.restoreFocusTarget.focus === "function" - ) { - this.restoreFocusTarget.focus(); - } - this.restoreFocusTarget = null; - } - - escapeHtml(text) { - const div = document.createElement("div"); - div.textContent = text == null ? "" : text; - return div.innerHTML; - } -} - -// Register the custom element -customElements.define("navigation-search", NavigationSearch); diff --git a/datasette/static/table.js b/datasette/static/table.js index e9115453..909eebf3 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -1,6 +1,13 @@ var DROPDOWN_HTML = `
    -
      +

      `; @@ -10,509 +17,54 @@ var DROPDOWN_ICON_SVG = ` `; -var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog"; -var setColumnTypeDialogState = null; - -function getParams() { - return new URLSearchParams(location.search); -} - -function paramsToUrl(params) { - var s = params.toString(); - return s ? "?" + s : location.pathname; -} - -function sortDescUrl(column) { - var params = getParams(); - params.set("_sort_desc", column); - params.delete("_sort"); - params.delete("_next"); - return paramsToUrl(params); -} - -function sortAscUrl(column) { - var params = getParams(); - params.set("_sort", column); - params.delete("_sort_desc"); - params.delete("_next"); - return paramsToUrl(params); -} - -function facetUrl(column) { - var params = getParams(); - params.append("_facet", column); - return paramsToUrl(params); -} - -function hideColumnUrl(column) { - var params = getParams(); - params.append("_nocol", column); - return paramsToUrl(params); -} - -function showAllColumnsUrl() { - var params = getParams(); - params.delete("_nocol"); - params.delete("_col"); - return paramsToUrl(params); -} - -function notBlankUrl(column) { - var params = getParams(); - params.set(`${column}__notblank`, "1"); - return paramsToUrl(params); -} - -function getDisplayedFacets() { - return Array.from(document.querySelectorAll(".facet-info")).map( - (el) => el.dataset.column, - ); -} - -function getColumnClassName(th) { - return Array.from(th.classList).find((className) => - className.startsWith("col-"), - ); -} - -function getColumnCells(th) { - var table = th.closest("table"); - var columnClassName = getColumnClassName(th); - if (!table || !columnClassName) { - return []; - } - return Array.from(table.querySelectorAll("td." + columnClassName)); -} - -function getColumnMeta(th) { - return { - columnName: th.dataset.column, - columnNotNull: th.dataset.columnNotNull === "1", - columnType: th.dataset.columnType, - isPk: th.dataset.isPk === "1", - }; -} - -function getColumnTypeText(th) { - var columnType = th.dataset.columnType; - if (!columnType) { - return null; - } - var notNull = th.dataset.columnNotNull === "1" ? " NOT NULL" : ""; - return `Type: ${columnType.toUpperCase()}${notNull}`; -} - -function getSetColumnTypeData() { - return window._setColumnTypeData || null; -} - -function getSetColumnTypeConfig(column) { - var data = getSetColumnTypeData(); - if (!data || !data.columns) { - return null; - } - return data.columns[column] || null; -} - -function canSetColumnType() { - return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch); -} - -function setColumnTypeActionLabel(column) { - var columnConfig = getSetColumnTypeConfig(column); - if (!columnConfig) { - return null; - } - return columnConfig.current - ? `Custom type: ${columnConfig.current.type}` - : "Set custom type"; -} - -function createSetColumnTypeOption(value, name, description, checked) { - var label = document.createElement("label"); - label.className = "set-column-type-option"; - - var input = document.createElement("input"); - input.type = "radio"; - input.name = "set-column-type-choice"; - input.value = value; - input.checked = checked; - - var content = document.createElement("span"); - content.className = "set-column-type-option-content"; - - var title = document.createElement("span"); - title.className = "set-column-type-option-name"; - title.textContent = name; - - var detail = document.createElement("span"); - detail.className = "set-column-type-option-description"; - detail.textContent = description; - - content.appendChild(title); - content.appendChild(detail); - label.appendChild(input); - label.appendChild(content); - return label; -} - -function setSetColumnTypeDialogBusy(state, isBusy) { - state.isBusy = isBusy; - state.saveButton.disabled = isBusy; - state.cancelButton.disabled = isBusy; - Array.from( - state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'), - ).forEach(function (input) { - input.disabled = isBusy; - }); - state.saveButton.textContent = isBusy ? "Saving..." : "Save"; -} - -function clearSetColumnTypeDialogError(state) { - state.error.hidden = true; - state.error.textContent = ""; -} - -function showSetColumnTypeDialogError(state, message) { - state.error.hidden = false; - state.error.textContent = message; -} - -function ensureSetColumnTypeDialog() { - if (setColumnTypeDialogState) { - return setColumnTypeDialogState; - } - if (!window.HTMLDialogElement) { - return null; - } - - var dialog = document.createElement("dialog"); - dialog.id = SET_COLUMN_TYPE_DIALOG_ID; - dialog.className = "set-column-type-dialog"; - dialog.setAttribute("aria-labelledby", "set-column-type-title"); - dialog.innerHTML = ` -
      - Set custom type - -
      -

      - -
      -
      - - - -
      - `; - document.body.appendChild(dialog); - - setColumnTypeDialogState = { - dialog: dialog, - meta: dialog.querySelector(".modal-meta"), - status: dialog.querySelector(".set-column-type-status"), - error: dialog.querySelector(".set-column-type-error"), - optionsWrap: dialog.querySelector(".set-column-type-options"), - footerInfo: dialog.querySelector(".footer-info"), - cancelButton: dialog.querySelector(".set-column-type-cancel"), - saveButton: dialog.querySelector(".set-column-type-save"), - currentColumn: null, - currentConfig: null, - isBusy: false, - }; - - setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - if (!setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - if (setColumnTypeDialogState.isBusy) { - ev.preventDefault(); - } - }); - - dialog.addEventListener("close", function () { - clearSetColumnTypeDialogError(setColumnTypeDialogState); - setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); - }); - - setColumnTypeDialogState.saveButton.addEventListener("click", async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; - - if (selectedType === currentType) { - state.dialog.close(); - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); - } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }); - - return setColumnTypeDialogState; -} - -function openSetColumnTypeDialog(th) { - var column = th.dataset.column; - var columnConfig = getSetColumnTypeConfig(column); - if (!columnConfig) { - return; - } - - var state = ensureSetColumnTypeDialog(); - if (!state) { - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, false); - state.currentColumn = column; - state.currentConfig = columnConfig; - state.status.textContent = `Column: ${column}`; - state.meta.textContent = getColumnTypeText(th) || "Type unavailable"; - state.footerInfo.textContent = columnConfig.current - ? `Current custom type: ${columnConfig.current.type}` - : "No custom type set."; - state.optionsWrap.innerHTML = ""; - - var currentType = columnConfig.current ? columnConfig.current.type : ""; - state.optionsWrap.appendChild( - createSetColumnTypeOption( - "", - "No custom type", - "Use standard Datasette rendering without a custom type.", - currentType === "", - ), - ); - - columnConfig.options.forEach(function (option) { - state.optionsWrap.appendChild( - createSetColumnTypeOption( - option.name, - option.name, - option.description, - option.name === currentType, - ), - ); - }); - - if (!columnConfig.options.length) { - var emptyState = document.createElement("p"); - emptyState.className = "set-column-type-empty"; - emptyState.textContent = - "No registered custom types are compatible with this SQLite type."; - state.optionsWrap.appendChild(emptyState); - } - - if (!state.dialog.open) { - state.dialog.showModal(); - } - var selectedOption = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - if (selectedOption) { - selectedOption.focus(); - } else { - state.saveButton.focus(); - } -} - -function canChooseColumns() { - return !!( - document.querySelector("column-chooser") && window._columnChooserData - ); -} - -function shouldShowShowAllColumns() { - var params = getParams(); - return params.getAll("_nocol").length || params.getAll("_col").length; -} - -function hasMultipleVisibleColumns(manager) { - return ( - Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( - (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", - ).length > 1 - ); -} - -function buildColumnActionItems(manager, th, options) { - options = options || {}; - var params = getParams(); - var column = th.dataset.column; - var columnActions = []; - var isSortable = !!th.querySelector("a"); - var isFirstColumn = th.parentElement.querySelector("th:first-of-type") === th; - var isSinglePk = - th.dataset.isPk === "1" && - document.querySelectorAll('th[data-is-pk="1"]').length === 1; - var hasBlankValues = getColumnCells(th).some( - (el) => el.innerText.trim() === "", - ); - - if (isSortable && params.get("_sort") !== column) { - columnActions.push({ - label: "Sort ascending", - href: sortAscUrl(column), - }); - } - - if (isSortable && params.get("_sort_desc") !== column) { - columnActions.push({ - label: "Sort descending", - href: sortDescUrl(column), - }); - } - - if ( - DATASETTE_ALLOW_FACET && - !isFirstColumn && - !getDisplayedFacets().includes(column) && - !isSinglePk - ) { - columnActions.push({ - label: "Facet by this", - href: facetUrl(column), - }); - } - - if (options.includeChooseColumns && canChooseColumns()) { - columnActions.push({ - label: "Choose columns", - href: "#", - onClick: - options.onChooseColumns || - function (ev) { - ev.preventDefault(); - openColumnChooser(); - }, - }); - } - - if (canSetColumnType() && getSetColumnTypeConfig(column)) { - columnActions.push({ - label: setColumnTypeActionLabel(column), - href: "#", - onClick: - options.onSetColumnType || - function (ev) { - ev.preventDefault(); - window.setTimeout(function () { - openSetColumnTypeDialog(th); - }, 0); - }, - }); - } - - if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) { - columnActions.push({ - label: "Hide this column", - href: hideColumnUrl(column), - }); - } - - if (options.includeShowAllColumns && shouldShowShowAllColumns()) { - columnActions.push({ - label: "Show all columns", - href: showAllColumnsUrl(), - }); - } - - if (params.get(`${column}__notblank`) !== "1" && hasBlankValues) { - columnActions.push({ - label: "Show not-blank rows", - href: notBlankUrl(column), - }); - } - - return columnActions.concat(manager.makeColumnActions(getColumnMeta(th))); -} - -function buildColumnActionState(manager, th, options) { - return { - column: th.dataset.column, - columnDescription: th.dataset.columnDescription || null, - columnMeta: getColumnMeta(th), - columnTypeText: getColumnTypeText(th), - actionItems: buildColumnActionItems(manager, th, options), - }; -} - -function initializeColumnActions(manager) { - manager.columnActions = { - buildColumnActionState: function (th, options) { - return buildColumnActionState(manager, th, options); - }, - buildColumnActionItems: function (th, options) { - return buildColumnActionItems(manager, th, options); - }, - canChooseColumns: canChooseColumns, - facetUrl: facetUrl, - getColumnMeta: getColumnMeta, - getColumnTypeText: getColumnTypeText, - hideColumnUrl: hideColumnUrl, - notBlankUrl: notBlankUrl, - shouldShowShowAllColumns: shouldShowShowAllColumns, - showAllColumnsUrl: showAllColumnsUrl, - sortAscUrl: sortAscUrl, - sortDescUrl: sortDescUrl, - }; -} - -function renderActionLink(itemConfig) { - var newLink = document.createElement("a"); - newLink.textContent = itemConfig.label; - newLink.href = itemConfig.href || "#"; - if (itemConfig.onClick) { - newLink.addEventListener("click", itemConfig.onClick); - } - return newLink; -} - /** Main initialization function for Datasette Table interactions */ const initDatasetteTable = function (manager) { // Feature detection if (!window.URLSearchParams) { return; } + function getParams() { + return new URLSearchParams(location.search); + } + function paramsToUrl(params) { + var s = params.toString(); + return s ? "?" + s : location.pathname; + } + function sortDescUrl(column) { + var params = getParams(); + params.set("_sort_desc", column); + params.delete("_sort"); + params.delete("_next"); + return paramsToUrl(params); + } + function sortAscUrl(column) { + var params = getParams(); + params.set("_sort", column); + params.delete("_sort_desc"); + params.delete("_next"); + return paramsToUrl(params); + } + function facetUrl(column) { + var params = getParams(); + params.append("_facet", column); + return paramsToUrl(params); + } + function hideColumnUrl(column) { + var params = getParams(); + params.append("_nocol", column); + return paramsToUrl(params); + } + function showAllColumnsUrl() { + var params = getParams(); + params.delete("_nocol"); + params.delete("_col"); + return paramsToUrl(params); + } + function notBlankUrl(column) { + var params = getParams(); + params.set(`${column}__notblank`, "1"); + return paramsToUrl(params); + } function closeMenu() { menu.style.display = "none"; menu.classList.remove("anim-scale-in"); @@ -544,41 +96,87 @@ const initDatasetteTable = function (manager) { var rect = th.getBoundingClientRect(); var menuTop = rect.bottom + window.scrollY; var menuLeft = rect.left + window.scrollX; - var actionState = manager.columnActions.buildColumnActionState(th, { - includeChooseColumns: true, - includeShowAllColumns: true, - onChooseColumns: function (ev) { - ev.preventDefault(); - closeMenu(); - openColumnChooser(); - }, - onSetColumnType: function (ev) { - ev.preventDefault(); - closeMenu(); - window.setTimeout(function () { - openSetColumnTypeDialog(th); - }, 0); - }, - }); - var menuList = menu.querySelector("ul.dropdown-actions"); - menuList.innerHTML = ""; - actionState.actionItems.forEach((itemConfig) => { - var menuItem = document.createElement("li"); - menuItem.appendChild(renderActionLink(itemConfig)); - menuList.appendChild(menuItem); - }); - + var column = th.getAttribute("data-column"); + var params = getParams(); + var sort = menu.querySelector("a.dropdown-sort-asc"); + var sortDesc = menu.querySelector("a.dropdown-sort-desc"); + var facetItem = menu.querySelector("a.dropdown-facet"); + var notBlank = menu.querySelector("a.dropdown-not-blank"); + var hideColumn = menu.querySelector("a.dropdown-hide-column"); + var showAllColumns = menu.querySelector("a.dropdown-show-all-columns"); + if (params.get("_sort") == column) { + sort.parentNode.style.display = "none"; + } else { + sort.parentNode.style.display = "block"; + sort.setAttribute("href", sortAscUrl(column)); + } + if (params.get("_sort_desc") == column) { + sortDesc.parentNode.style.display = "none"; + } else { + sortDesc.parentNode.style.display = "block"; + sortDesc.setAttribute("href", sortDescUrl(column)); + } + /* Show hide columns options */ + if (params.get("_nocol") || params.get("_col")) { + showAllColumns.parentNode.style.display = "block"; + showAllColumns.setAttribute("href", showAllColumnsUrl()); + } else { + showAllColumns.parentNode.style.display = "none"; + } + if (th.getAttribute("data-is-pk") != "1") { + hideColumn.parentNode.style.display = "block"; + hideColumn.setAttribute("href", hideColumnUrl(column)); + } else { + hideColumn.parentNode.style.display = "none"; + } + /* Only show "Facet by this" if it's not the first column, not selected, + not a single PK and the Datasette allow_facet setting is True */ + var displayedFacets = Array.from( + document.querySelectorAll(".facet-info") + ).map((el) => el.dataset.column); + var isFirstColumn = + th.parentElement.querySelector("th:first-of-type") == th; + var isSinglePk = + th.getAttribute("data-is-pk") == "1" && + document.querySelectorAll('th[data-is-pk="1"]').length == 1; + if ( + !DATASETTE_ALLOW_FACET || + isFirstColumn || + displayedFacets.includes(column) || + isSinglePk + ) { + facetItem.parentNode.style.display = "none"; + } else { + facetItem.parentNode.style.display = "block"; + facetItem.setAttribute("href", facetUrl(column)); + } + /* Show notBlank option if not selected AND at least one visible blank value */ + var tdsForThisColumn = Array.from( + th.closest("table").querySelectorAll("td." + th.className) + ); + if ( + params.get(`${column}__notblank`) != "1" && + tdsForThisColumn.filter((el) => el.innerText.trim() == "").length + ) { + notBlank.parentNode.style.display = "block"; + notBlank.setAttribute("href", notBlankUrl(column)); + } else { + notBlank.parentNode.style.display = "none"; + } var columnTypeP = menu.querySelector(".dropdown-column-type"); - if (actionState.columnTypeText) { + var columnType = th.dataset.columnType; + var notNull = th.dataset.columnNotNull == 1 ? " NOT NULL" : ""; + + if (columnType) { columnTypeP.style.display = "block"; - columnTypeP.innerText = actionState.columnTypeText; + columnTypeP.innerText = `Type: ${columnType.toUpperCase()}${notNull}`; } else { columnTypeP.style.display = "none"; } var columnDescriptionP = menu.querySelector(".dropdown-column-description"); - if (actionState.columnDescription) { - columnDescriptionP.innerText = actionState.columnDescription; + if (th.dataset.columnDescription) { + columnDescriptionP.innerText = th.dataset.columnDescription; columnDescriptionP.style.display = "block"; } else { columnDescriptionP.style.display = "none"; @@ -589,6 +187,37 @@ const initDatasetteTable = function (manager) { menu.style.display = "block"; menu.classList.add("anim-scale-in"); + // Custom menu items on each render + // Plugin hook: allow adding JS-based additional menu items + const columnActionsPayload = { + columnName: th.dataset.column, + columnNotNull: th.dataset.columnNotNull === '1', + columnType: th.dataset.columnType, + isPk: th.dataset.isPk === '1' + }; + const columnItemConfigs = manager.makeColumnActions(columnActionsPayload); + + const menuList = menu.querySelector('ul'); + columnItemConfigs.forEach(itemConfig => { + // Remove items from previous render. We assume entries have unique labels. + const existingItems = menuList.querySelectorAll(`li`); + Array.from(existingItems).filter(item => item.innerText === itemConfig.label).forEach(node => { + node.remove(); + }); + + const newLink = document.createElement('a'); + newLink.textContent = itemConfig.label; + newLink.href = itemConfig.href ?? '#'; + if (itemConfig.onClick) { + newLink.onclick = itemConfig.onClick; + } + + // Attach new elements to DOM + const menuItem = document.createElement('li'); + menuItem.appendChild(newLink); + menuList.appendChild(menuItem); + }); + // Measure width of menu and adjust position if too far right const menuWidth = menu.offsetWidth; const windowWidth = window.innerWidth; @@ -596,17 +225,17 @@ const initDatasetteTable = function (manager) { menu.style.left = windowWidth - menuWidth - 20 + "px"; } // Align menu .hook arrow with the column cog icon - const hook = menu.querySelector(".hook"); - const icon = th.querySelector(".dropdown-menu-icon"); + const hook = menu.querySelector('.hook'); + const icon = th.querySelector('.dropdown-menu-icon'); const iconRect = icon.getBoundingClientRect(); - const hookLeft = iconRect.left - menuLeft + 1 + "px"; + const hookLeft = (iconRect.left - menuLeft + 1) + 'px'; hook.style.left = hookLeft; // Move the whole menu right if the hook is too far right const menuRect = menu.getBoundingClientRect(); if (iconRect.right > menuRect.right) { - menu.style.left = iconRect.right - menuWidth + "px"; + menu.style.left = (iconRect.right - menuWidth) + 'px'; // And move hook tip as well - hook.style.left = menuWidth - 13 + "px"; + hook.style.left = (menuWidth - 13) + 'px'; } } @@ -621,9 +250,7 @@ const initDatasetteTable = function (manager) { menu.style.display = "none"; document.body.appendChild(menu); - var ths = Array.from( - document.querySelectorAll(manager.selectors.tableHeaders), - ); + var ths = Array.from(document.querySelectorAll(manager.selectors.tableHeaders)); ths.forEach((th) => { if (!th.querySelector("a")) { return; @@ -637,9 +264,9 @@ const initDatasetteTable = function (manager) { /* Add x buttons to the filter rows */ function addButtonsToFilterRows(manager) { var x = "✖"; - var rows = Array.from( - document.querySelectorAll(manager.selectors.filterRow), - ).filter((el) => el.querySelector(".filter-op")); + var rows = Array.from(document.querySelectorAll(manager.selectors.filterRow)).filter((el) => + el.querySelector(".filter-op") + ); rows.forEach((row) => { var a = document.createElement("a"); a.setAttribute("href", "#"); @@ -660,18 +287,18 @@ function addButtonsToFilterRows(manager) { a.style.display = "none"; } }); -} +}; /* Set up datalist autocomplete for filter values */ function initAutocompleteForFilterValues(manager) { function createDataLists() { var facetResults = document.querySelectorAll( - manager.selectors.facetResults, + manager.selectors.facetResults ); Array.from(facetResults).forEach(function (facetResult) { // Use link text from all links in the facet result var links = Array.from( - facetResult.querySelectorAll("li:not(.facet-truncated) a"), + facetResult.querySelectorAll("li:not(.facet-truncated) a") ); // Create a datalist element var datalist = document.createElement("datalist"); @@ -697,57 +324,12 @@ function initAutocompleteForFilterValues(manager) { .setAttribute("list", "datalist-" + event.target.value); } }); -} - -/** Open the column-chooser web component */ -function openColumnChooser() { - var chooser = document.querySelector("column-chooser"); - var data = window._columnChooserData; - if (!chooser || !data) return; - - var nonPkColumns = data.allColumns.filter(function (col) { - return data.primaryKeys.indexOf(col) === -1; - }); - var selected = data.selectedColumns.filter(function (col) { - return data.primaryKeys.indexOf(col) === -1; - }); - - chooser.open({ - columns: nonPkColumns, - selected: selected, - onApply: function (cols) { - var params = new URLSearchParams(location.search); - params.delete("_col"); - params.delete("_nocol"); - params.delete("_next"); - - if (cols.length === nonPkColumns.length) { - // Check if order matches original - if so, no params needed - var orderMatches = cols.every(function (col, i) { - return col === nonPkColumns[i]; - }); - if (!orderMatches) { - cols.forEach(function (col) { - params.append("_col", col); - }); - } - } else { - cols.forEach(function (col) { - params.append("_col", col); - }); - } - var qs = params.toString(); - location.href = qs ? "?" + qs : location.pathname; - }, - }); -} +}; // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; - initializeColumnActions(manager); - // Main table initDatasetteTable(manager); diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py deleted file mode 100644 index a6123daa..00000000 --- a/datasette/stored_queries.py +++ /dev/null @@ -1,581 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -from typing import Any, Iterable - -from .utils import tilde_encode, urlsafe_components - -UNCHANGED = object() - - -QUERY_OPTION_FIELDS = ( - "hide_sql", - "fragment", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", -) - - -@dataclass -class StoredQuery: - database: str - name: str - sql: str - title: str | None - description: str | None - description_html: str | None - hide_sql: bool - fragment: str | None - parameters: list[str] - is_write: bool - is_private: bool - is_trusted: bool - source: str - owner_id: str | None - on_success_message: str | None - on_success_message_sql: str | None - on_success_redirect: str | None - on_error_message: str | None - on_error_redirect: str | None - private: bool | None = None - - -@dataclass -class StoredQueryPage: - queries: list[StoredQuery] - next: str | None - has_more: bool - limit: int - - -def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: - data = { - "database": query.database, - "name": query.name, - "sql": query.sql, - "title": query.title, - "description": query.description, - "description_html": query.description_html, - "hide_sql": query.hide_sql, - "fragment": query.fragment, - "params": list(query.parameters), - "parameters": list(query.parameters), - "is_write": query.is_write, - "is_private": query.is_private, - "is_trusted": query.is_trusted, - "source": query.source, - "owner_id": query.owner_id, - "on_success_message": query.on_success_message, - "on_success_message_sql": query.on_success_message_sql, - "on_success_redirect": query.on_success_redirect, - "on_error_message": query.on_error_message, - "on_error_redirect": query.on_error_redirect, - } - if query.private is not None: - data["private"] = query.private - return data - - -def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: - return { - "queries": [stored_query_to_dict(query) for query in page.queries], - "next": page.next, - "has_more": page.has_more, - "limit": page.limit, - } - - -async def save_queries_from_config(datasette: Any) -> None: - # Apply configured query entries from datasette.yaml to the internal table. - await datasette.get_internal_database().execute_write( - "DELETE FROM queries WHERE source = 'config'" - ) - for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if not isinstance(query_config, dict): - query_config = {"sql": query_config} - await datasette.add_query( - dbname, - query_name, - query_config["sql"], - title=query_config.get("title"), - description=query_config.get("description"), - description_html=query_config.get("description_html"), - hide_sql=bool(query_config.get("hide_sql")), - fragment=query_config.get("fragment"), - parameters=query_config.get("params"), - is_write=bool(query_config.get("write")), - is_trusted=bool(query_config.get("is_trusted", True)), - source="config", - on_success_message=query_config.get("on_success_message"), - on_success_message_sql=query_config.get("on_success_message_sql"), - on_success_redirect=query_config.get("on_success_redirect"), - on_error_message=query_config.get("on_error_message"), - on_error_redirect=query_config.get("on_error_redirect"), - ) - - -def query_row_to_stored_query( - row: Any, private: bool | None = None -) -> StoredQuery | None: - if row is None: - return None - parameters = json.loads(row["parameters"] or "[]") - options = json.loads(row["options"] or "{}") - return StoredQuery( - database=row["database_name"], - name=row["name"], - sql=row["sql"], - title=row["title"], - description=row["description"], - description_html=row["description_html"], - hide_sql=bool(options.get("hide_sql")), - fragment=options.get("fragment"), - parameters=parameters, - is_write=bool(row["is_write"]), - is_private=bool(row["is_private"]), - is_trusted=bool(row["is_trusted"]), - source=row["source"], - owner_id=row["owner_id"], - on_success_message=options.get("on_success_message"), - on_success_message_sql=options.get("on_success_message_sql"), - on_success_redirect=options.get("on_success_redirect"), - on_error_message=options.get("on_error_message"), - on_error_redirect=options.get("on_error_redirect"), - private=private, - ) - - -def query_options_json(options: dict[str, Any]) -> str: - options_dict = {} - for field in QUERY_OPTION_FIELDS: - value = options.get(field) - if field == "hide_sql": - if value: - options_dict[field] = True - elif value is not None: - options_dict[field] = value - return json.dumps(options_dict, sort_keys=True) - - -async def add_query( - datasette: Any, - database: str, - name: str, - sql: str, - *, - title: str | None = None, - description: str | None = None, - description_html: str | None = None, - hide_sql: bool = False, - fragment: str | None = None, - parameters: Iterable[str] | None = None, - is_write: bool = False, - is_private: bool = False, - is_trusted: bool = False, - source: str = "plugin", - owner_id: str | None = None, - on_success_message: str | None = None, - on_success_message_sql: str | None = None, - on_success_redirect: str | None = None, - on_error_message: str | None = None, - on_error_redirect: str | None = None, - replace: bool = True, -) -> None: - parameters_json = json.dumps(list(parameters or [])) - options_json = query_options_json( - { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - ) - sql_statement = """ - INSERT INTO queries ( - database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_private, is_trusted, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - if replace: - sql_statement += """ - ON CONFLICT(database_name, name) DO UPDATE SET - sql = excluded.sql, - title = excluded.title, - description = excluded.description, - description_html = excluded.description_html, - options = excluded.options, - parameters = excluded.parameters, - is_write = excluded.is_write, - is_private = excluded.is_private, - is_trusted = excluded.is_trusted, - source = excluded.source, - owner_id = excluded.owner_id, - updated_at = CURRENT_TIMESTAMP - """ - await datasette.get_internal_database().execute_write( - sql_statement, - [ - database, - name, - sql, - title, - description, - description_html, - options_json, - parameters_json, - int(bool(is_write)), - int(bool(is_private)), - int(bool(is_trusted)), - source, - owner_id, - ], - ) - - -async def update_query( - datasette: Any, - database: str, - name: str, - *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, -) -> None: - fields = { - "sql": sql, - "title": title, - "description": description, - "description_html": description_html, - "parameters": parameters, - "is_write": is_write, - "is_private": is_private, - "is_trusted": is_trusted, - "source": source, - "owner_id": owner_id, - } - option_fields = { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - updates = [] - params = [] - for field, value in fields.items(): - if value is UNCHANGED: - continue - if field in {"is_write", "is_private", "is_trusted"}: - value = int(bool(value)) - elif field == "parameters": - value = json.dumps(list(value or [])) - updates.append(f"{field} = ?") - params.append(value) - changed_options = { - field: value for field, value in option_fields.items() if value is not UNCHANGED - } - if changed_options: - rows = await datasette.get_internal_database().execute( - """ - SELECT options FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - row = rows.first() - options = json.loads(row["options"] or "{}") if row is not None else {} - for field, value in changed_options.items(): - if field == "hide_sql": - if value: - options[field] = True - else: - options.pop(field, None) - elif value is None: - options.pop(field, None) - else: - options[field] = value - updates.append("options = ?") - params.append(json.dumps(options, sort_keys=True)) - if not updates: - return - updates.append("updated_at = CURRENT_TIMESTAMP") - params.extend([database, name]) - await datasette.get_internal_database().execute_write( - """ - UPDATE queries - SET {} - WHERE database_name = ? AND name = ? - """.format(", ".join(updates)), - params, - ) - - -async def remove_query( - datasette: Any, database: str, name: str, source: str | None = None -) -> None: - sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" - params = [database, name] - if source is not None: - sql += " AND source = ?" - params.append(source) - await datasette.get_internal_database().execute_write(sql, params) - - -async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: - rows = await datasette.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - return query_row_to_stored_query(rows.first()) - - -async def count_queries( - datasette: Any, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, -) -> int: - allowed_sql, allowed_params = await datasette.allowed_resources_sql( - action="view-query", - actor=actor, - parent=database, - ) - params = dict(allowed_params) - where_clauses = [] - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - row = ( - await datasette.get_internal_database().execute( - """ - SELECT count(*) AS count - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - """.format( - allowed_sql=allowed_sql, - where=" AND ".join(where_clauses) or "1 = 1", - ), - params, - ) - ).first() - return row["count"] - - -async def list_queries( - datasette: Any, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - limit: int = 50, - cursor: str | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - include_private: bool = False, -) -> StoredQueryPage: - limit = min(max(1, int(limit)), 1000) - allowed_sql, allowed_params = await datasette.allowed_resources_sql( - action="view-query", - actor=actor, - parent=database, - include_is_private=include_private, - ) - params = dict(allowed_params) - params.update({"limit": limit + 1}) - sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = [] - order_by = "q.database_name, sort_key, q.name" - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - order_by = "sort_key, q.name" - - if cursor: - try: - components = urlsafe_components(cursor) - except ValueError: - components = [] - if database is None and len(components) == 3: - where_clauses.append(""" - ( - q.database_name > :cursor_database - OR ( - q.database_name = :cursor_database - AND ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_database"] = components[0] - params["cursor_sort_key"] = components[1] - params["cursor_name"] = components[2] - elif database is not None and len(components) == 2: - where_clauses.append(""" - ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_sort_key"] = components[0] - params["cursor_name"] = components[1] - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - private_select = ", allowed.is_private AS private" if include_private else "" - rows = list( - ( - await datasette.get_internal_database().execute( - """ - SELECT q.*, {sort_key_sql} AS sort_key{private_select} - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - ORDER BY {order_by} - LIMIT :limit - """.format( - allowed_sql=allowed_sql, - private_select=private_select, - sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses) or "1 = 1", - order_by=order_by, - ), - params, - ) - ).rows - ) - has_more = len(rows) > limit - if has_more: - rows = rows[:limit] - - queries = [] - for row in rows: - query = query_row_to_stored_query( - row, private=bool(row["private"]) if include_private else None - ) - assert query is not None - queries.append(query) - - next_token = None - if has_more and rows: - last_row = rows[-1] - if database is None: - next_token = "{},{},{}".format( - tilde_encode(last_row["database_name"]), - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - else: - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - return StoredQueryPage( - queries=queries, - next=next_token, - has_more=has_more, - limit=limit, - ) diff --git a/datasette/templates/_action_menu.html b/datasette/templates/_action_menu.html index 1ae8c173..7d1d4a55 100644 --- a/datasette/templates/_action_menu.html +++ b/datasette/templates/_action_menu.html @@ -1,7 +1,7 @@ {% if action_links %}
      -
      {% for column in display_columns %} -
      + {% if not column.sortable %} {{ column.name }} {% else %} @@ -31,7 +31,6 @@
      -{% endif %} -{% if not display_rows %} +{% else %}

      0 records

      {% endif %} diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..610417d2 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -33,9 +33,6 @@ p.message-warning {

      Debug allow rules

      -{% set current_tab = "allow_debug" %} -{% include "_permissions_debug_tabs.html" %} -

      Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

      diff --git a/datasette/templates/base.html b/datasette/templates/base.html index e1767deb..0b2def5a 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -20,7 +20,7 @@