diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 3fc83438..cf9b25a7 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,24 +14,46 @@ jobs: deploy: runs-on: ubuntu-latest steps: + - name: Check deployment prerequisites + id: deployment-prerequisites + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} + run: | + missing=() + for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do + if [[ -z "${!variable:-}" ]]; then + missing+=("$variable") + fi + done + if (( ${#missing[@]} )); then + echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi - name: Check out datasette + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Run tests - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -39,14 +61,18 @@ jobs: fixtures-metadata.json \ plugins \ --extra-db-filename extra_database.db + # Package the config with the plugins, excluding test-only plugin secrets + # that reference temporary files outside the deployed container. + jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json - name: Build docs.db - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -58,6 +84,7 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Deploy to docs as well (only for main) - if: ${{ github.ref == 'refs/heads/main' }} + - name: Upload latest documentation database to S3 (only for main) + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - # Deploy docs.db to a different service - datasette publish cloudrun docs.db \ - --branch=$GITHUB_SHA \ - --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1" \ - --service=datasette-docs-latest + # Keep development documentation separate from the stable release database. + s3-credentials put-object datasette-docs latest/docs.db docs.db \ + --content-type application/octet-stream diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f5b8dbf6..85369f6c 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,9 +2,15 @@ name: Playwright on: push: + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index d92ab82b..fa7ec6aa 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -1,6 +1,15 @@ name: Check JavaScript for conformance with Prettier -on: [push] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 21ed4c12..232a34c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [created] + types: [published] permissions: contents: read @@ -51,6 +51,8 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 + # After the first non-prerelease 1.0 release, disable this job on 0.65.x, + # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -66,26 +68,20 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy stable-docs.datasette.io to Cloud Run + - name: Upload stable documentation database to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - datasette publish cloudrun docs.db \ - --service=datasette-docs-stable + s3-credentials put-object datasette-docs docs.db docs.db \ + --content-type application/octet-stream deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 58635025..aa35338f 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -1,6 +1,15 @@ name: Check spelling in documentation -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml deleted file mode 100644 index e9bd4bab..00000000 --- a/.github/workflows/test-coverage.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Calculate test coverage - -on: - push: - branches: - - main - pull_request: - branches: - - main -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Check out datasette - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install . --group dev - python -m pip install pytest-cov - - name: Run tests - run: |- - ls -lah - cat .coveragerc - pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x - ls -lah - - name: Upload coverage report - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5e81ed82..449855f3 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper on: push: + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 2fdb3a40..700f3cce 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -1,6 +1,15 @@ name: Test SQLite versions -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -12,10 +21,10 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.13"] sqlite-version: [ #"3", # latest version - "3.46", + #"3.46", #"3.45", #"3.27", #"3.26", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 751eedfd..8176a630 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,15 @@ name: Test -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -11,16 +20,20 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + include: + - python-version: "3.14" + coverage: true steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml + check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) @@ -28,12 +41,27 @@ jobs: run: | pip install . --group dev pip freeze + - name: Install pytest-cov + if: ${{ matrix.coverage }} + run: pip install pytest-cov - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" + if [ "${{ matrix.coverage }}" = "true" ]; then + COV="--cov=datasette --cov-config=.coveragerc" + pytest -n auto -m "not serial" $COV --cov-report= + pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term + else + pytest -n auto -m "not serial" + pytest -m "serial" + fi # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh + - name: Upload coverage report + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml - name: Black run: | black --version diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..58287dd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.11-slim-bookworm AS build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/Justfile b/Justfile index 6ffff870..d1b69378 100644 --- a/Justfile +++ b/Justfile @@ -49,13 +49,18 @@ export DATASETTE_SECRET := "not_a_secret" uv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: cog blacken-docs +@docs: shots cog blacken-docs uv run make -C docs livehtml # Build docs as static HTML @docs-build: cog blacken-docs rm -rf docs/_build && cd docs && uv run make html +# Take any missing documentation screenshots defined in docs/shots.yml +@shots: + uv run --group shots shot-scraper install + cd docs && uv run --group shots shot-scraper multi shots.yml --no-clobber --reduced-motion --retina + # Apply Black @black: uv run black datasette tests diff --git a/README.md b/README.md index 393e8e5c..1f79778f 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/datasette/__init__.py b/datasette/__init__.py index e0022178..982dcc79 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,6 +1,7 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa +from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor # noqa from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa from datasette.utils.asgi import ( # noqa Forbidden, diff --git a/datasette/app.py b/datasette/app.py index c82ea075..3e4c5acf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import contextvars from collections.abc import Iterable, Sequence from typing import TYPE_CHECKING, Any @@ -28,7 +27,7 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx +import httpx2 from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, @@ -42,6 +41,7 @@ from jinja2.exceptions import TemplateNotFound from markupsafe import Markup, escape from . import stored_queries, write_sql +from .background_tasks import BackgroundTask, BackgroundTaskSupervisor from .column_types import SQLiteType from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted @@ -49,6 +49,16 @@ from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource +from .telemetry import ( + TelemetryMiddleware, + _in_datasette_client, + clamp_http_method, + register_datasette, + request_span, + tracer, + unregister_datasette, +) +from .telemetry_registry import HTTP_ROUTE, STARTUP from .tokens import TokenInvalid from .tracer import AsgiTracer from .url_builder import Urls @@ -145,6 +155,7 @@ from .views.stored_queries import ( ) from .views.table import ( TableAutocompleteView, + TableCountView, TableDropView, TableFragmentView, TableInsertView, @@ -164,8 +175,7 @@ app_root = Path(__file__).parent.parent logger = logging.getLogger(__name__) -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) +# _in_datasette_client is defined in telemetry.py to avoid a circular import class _DatasetteClientContext: @@ -315,7 +325,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action, parent, child) + return (actor_key, action.name, parent, action.normalize_child(child)) async def favicon(request, send): @@ -422,6 +432,7 @@ class Datasette: default_deny=False, ): self._startup_invoked = False + self._shutdown_invoked = False self._closed = False assert config_dir is None or isinstance( config_dir, Path @@ -453,8 +464,11 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this + self._setup_db_done = False + self._suppress_background_tasks = False try: self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -462,8 +476,10 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() else: raise + self._background_tasks = BackgroundTaskSupervisor(self) self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: @@ -635,6 +651,8 @@ class Datasette: self.root_enabled = False self.default_deny = default_deny self.client = DatasetteClient(self) + # Last, so metric callbacks never see a partially initialized instance + register_datasette(self) async def apply_metadata_json(self): # Apply any metadata entries from metadata.json to the internal tables @@ -775,57 +793,61 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # Register event classes - event_classes = [] - for hook in pm.hook.register_events(datasette=self): - extra_classes = await await_me_maybe(hook) - if extra_classes: - event_classes.extend(extra_classes) - self.event_classes = tuple(event_classes) + # Group spans created during startup under a single parent span + with tracer.start_as_current_span(STARTUP): + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): - if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): - raise StartupError(f"Duplicate action name: {action.name}") - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action + # Register actions, but watch out for duplicate name/abbr + action_names = {} + action_abbrs = {} + for hook in pm.hook.register_actions(datasette=self): + if hook: + for action in hook: + if ( + action.name in action_names + and action != action_names[action.name] + ): + raise StartupError(f"Duplicate action name: {action.name}") + if ( + action.abbr + and action.abbr in action_abbrs + and action != action_abbrs[action.abbr] + ): + raise StartupError(f"Duplicate action abbr: {action.abbr}") + action_names[action.name] = action + if action.abbr: + action_abbrs[action.abbr] = action + self.actions[action.name] = action - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError(f"Duplicate column type name: {ct_cls.name}") - self._column_types[ct_cls.name] = ct_cls + # Register column types (classes, not instances) + self._column_types = {} + for hook in pm.hook.register_column_types(datasette=self): + if hook: + for ct_cls in hook: + if ct_cls.name in self._column_types: + raise StartupError( + f"Duplicate column type name: {ct_cls.name}" + ) + self._column_types[ct_cls.name] = ct_cls - for hook in pm.hook.prepare_jinja2_environment( - env=self._jinja_env, datasette=self - ): - await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() - for hook in pm.hook.startup(datasette=self): - await await_me_maybe(hook) - self._startup_invoked = True + for hook in pm.hook.prepare_jinja2_environment( + env=self._jinja_env, datasette=self + ): + await await_me_maybe(hook) + # Ensure internal tables and metadata are populated before startup hooks + await self._refresh_schemas() + await self._save_queries_from_config() + # Load column_types from config into internal DB + await self._apply_column_types_config() + for hook in pm.hook.startup(datasette=self): + await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -958,6 +980,8 @@ class Datasette: if self._closed: return self._closed = True + # Stop reporting metrics before closing databases + unregister_datasette(self) first_exception = None dbs = list(self.databases.values()) + [self._internal_database] for db in dbs: @@ -1529,15 +1553,28 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: + # Extension loading is only enabled for as long as it takes to + # load the configured extensions. Leaving it enabled would let + # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) - else: - conn.execute("SELECT load_extension(?)", [extension]) + try: + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + if sys.version_info >= (3, 12): + conn.load_extension(path, entrypoint=entrypoint) + else: + # Connection.load_extension() only gained the + # entrypoint argument in Python 3.12 + conn.execute( + "SELECT load_extension(?, ?)", [path, entrypoint] + ) + else: + conn.load_extension(extension) + finally: + conn.enable_load_extension(False) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1730,8 +1767,145 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) + if action == "view-table": + sql, params = await self._apply_derived_table_permissions_to_sql( + sql, + params, + actor=actor, + parent=parent, + include_is_private=include_is_private, + ) return ResourcesSQL(sql, params) + async def _allowed_derived_table_source( + self, database, source, *, actor, dependencies + ): + """Check an immediate source, denying sources that are themselves derived.""" + if any( + TableResource.normalize_child(table) + == TableResource.normalize_child(source) + for table in dependencies + ): + return False + # The source has no dependency in this map. Evaluate its own permission + # and prerequisites without starting another dependency check. + verdicts = await self._allowed_many( + actions=["view-table"], + resource=TableResource(database, source), + actor=actor, + check_derived=False, + ) + return verdicts["view-table"] + + async def _apply_derived_table_permissions_to_sql( + self, + sql, + params, + *, + actor, + parent, + include_is_private, + ): + databases = ( + [(parent, self.databases[parent])] + if parent in self.databases + else ([] if parent is not None else list(self.databases.items())) + ) + dependency_maps = dict( + zip( + (name for name, _ in databases), + await asyncio.gather( + *(db.derived_table_dependencies() for _, db in databases) + ), + ) + ) + dependencies = [ + (database_name, child, source) + for database_name, dependency_map in dependency_maps.items() + for child, source in dependency_map.items() + ] + if not dependencies: + return sql, params + + sources = sorted( + {(database_name, source) for database_name, _, source in dependencies} + ) + actor_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=actor, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + actor_allowed = dict(zip(sources, actor_verdicts)) + + anonymous_allowed = {} + if include_is_private: + anonymous_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=None, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + anonymous_allowed = dict(zip(sources, anonymous_verdicts)) + + wrapped_params = dict(params) + derived_rows = [ + [ + database_name, + child, + int(actor_allowed[(database_name, source)]), + *( + [int(anonymous_allowed[(database_name, source)])] + if include_is_private + else [] + ), + ] + for database_name, child, source in dependencies + ] + derived_param = "_datasette_derived_permissions" + while derived_param in wrapped_params: + derived_param += "_" + wrapped_params[derived_param] = json.dumps(derived_rows) + + derived_columns = "parent, child, source_allowed" + select_columns = "allowed.parent, allowed.child, allowed.reason" + if include_is_private: + derived_columns += ", source_anonymous_allowed" + select_columns += ( + ", CASE WHEN derived.source_anonymous_allowed = 0 " + "THEN 1 ELSE allowed.is_private END AS is_private" + ) + wrapped_sql = f""" +WITH derived_permissions({derived_columns}) AS ( + SELECT + json_extract(value, '$[0]'), + json_extract(value, '$[1]'), + json_extract(value, '$[2]') + {", json_extract(value, '$[3]')" if include_is_private else ""} + FROM json_each(:{derived_param}) +), +allowed AS ( +{sql} +) +SELECT {select_columns} +FROM allowed +LEFT JOIN derived_permissions AS derived + ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE +WHERE COALESCE(derived.source_allowed, 1) = 1 +ORDER BY allowed.parent, allowed.child +""".strip() + return wrapped_sql, wrapped_params + async def allowed_resources( self, action: str, @@ -1934,6 +2108,12 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ + return await self._allowed_many( + actions=actions, resource=resource, actor=actor, check_derived=True + ) + + async def _allowed_many(self, *, actions, resource, actor, check_derived): + """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, @@ -1971,7 +2151,7 @@ class Datasette: to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, name, parent, child) + key = _permission_cache_key(actor, self.actions[name], parent, child) if key in cache: final[name] = cache[key] continue @@ -1987,6 +2167,28 @@ class Datasette: child=child, ) + if ( + check_derived + and "view-table" in to_check + and raw.get("view-table") + and isinstance(resource, TableResource) + and parent in self.databases + ): + dependencies = await self.databases[parent].derived_table_dependencies() + source = next( + ( + source + for table, source in dependencies.items() + if TableResource.normalize_child(table) + == TableResource.normalize_child(child) + ), + None, + ) + if source is not None: + raw["view-table"] = await self._allowed_derived_table_source( + parent, source, actor=actor, dependencies=dependencies + ) + def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2004,7 +2206,9 @@ class Datasette: # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[_permission_cache_key(actor, name, parent, child)] = final[name] + cache[ + _permission_cache_key(actor, self.actions[name], parent, child) + ] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2095,6 +2299,18 @@ class Datasette: from datasette.resources import TableResource other_table = fk["other_table"] + # Foreign key declarations can spell the target with different casing. + target_table = ( + await db.execute( + "select name from sqlite_master where type='table' and name=? collate nocase", + [other_table], + ) + ).first() + if target_table is None: + # SQLite accepts a foreign key to a table that does not exist, and + # linking to it would only lead to a 404 + return {} + other_table = target_table[0] other_column = fk["other_column"] if other_column is None: other_pks = await db.primary_keys(other_table) @@ -2278,6 +2494,21 @@ class Datasette: ) return d + def _tasks(self): + return { + "tasks": [ + { + "name": t.name, + "state": t.state, + "function": t.function, + "started_at": t.started_at, + "exception": repr(t.exception) if t.exception else None, + } + for t in self._background_tasks.tasks() + ], + "launched": self._background_tasks.launched, + } + def _actor(self, request): return {"actor": request.actor} @@ -2384,6 +2615,8 @@ class Datasette: datasette=self, ): extra_vars = await await_me_maybe(extra_vars) + if extra_vars is None: + continue assert isinstance( extra_vars, dict ), f"extra_vars is of type {type(extra_vars)}" @@ -2446,7 +2679,7 @@ class Datasette: ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + (24 * 60 * 60) + expires_at = int(time.time()) + expire_after data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2566,6 +2799,12 @@ class Datasette: ), r"/-/threads(\.(?Pjson))?$", ) + add_route( + JsonDataView.as_view( + self, "tasks.json", self._tasks, permission="permissions-debug" + ), + r"/-/tasks(\.(?Pjson))?$", + ) add_route( JsonDataView.as_view( self, @@ -2740,6 +2979,10 @@ class Datasette: TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) + add_route( + TableCountView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", + ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -2803,26 +3046,130 @@ class Datasette: raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) + async def _startup_sequence(self): + """Idempotently run the full startup sequence: table counts for + immutable databases, then invoke_startup(). Safe to call more than + once and safe to call concurrently - callers block until whichever + call got there first has finished. + + This is the single entry point used by both AsgiLifespan (so + real deployments finish startup before accepting requests) and + AsgiRunOnFirstRequest (the fallback for hosts that never send + lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and + `datasette serve` (cli.py) calls it too. The fast path below checks + both `_startup_invoked` and `_setup_db_done` - not just the former - + so that a bare `await ds.invoke_startup()` made by a caller ahead of + `_startup_sequence()` (which only sets `_startup_invoked`) can't + make this method skip the immutable-database table-count precompute. + """ + if self._startup_invoked and self._setup_db_done: + return + async with self._startup_lock: + if self._startup_invoked and self._setup_db_done: + return + if not self._setup_db_done: + # First time server starts up, calculate table counts for + # immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + self._setup_db_done = True + await self.invoke_startup() + + def add_background_task(self, func, name=None) -> BackgroundTask: + """Register a piece of supervised background work, typically from + a plugin's ``startup`` hook. + + ``func`` must be a coroutine function taking one positional + argument, the ``Datasette`` instance - core calls ``func(self)``. + Callable any time after ``__init__``: if background tasks haven't + launched yet (the common case - most callers are ``startup`` hooks, + which run before launch), this buffers the registration until they + do; if they've already launched (e.g. called from a request + handler after the server is up), the task starts immediately. + + Returns a :class:`~datasette.background_tasks.BackgroundTask` + handle (``.name``, ``.state``, ``.task``, ``.exception``, + ``.started_at``, ``.function``, ``.cancel()``). + + ``name`` defaults to ``func.__qualname__``; on a name collision a + ``-2``, ``-3``, ... suffix is appended, since names are how + ``/-/tasks`` and log messages identify work. + """ + return self._background_tasks.add(func, name=name) + + async def start_background_tasks(self): + """Run startup (if it hasn't run yet) and launch every registered + background task. + + Public entry point for tests, embedders, and headless CLIs (the + ``datasette-rss``-style ``fetch --due`` shape) that want supervised + background tasks without running a server - equivalent to what + happens automatically via ASGI lifespan / the first-request + fallback in a served deployment. + """ + await self.invoke_startup() + await self._background_tasks.launch_all() + + async def _launch_background_tasks(self): + """Idempotently launch every registered background task. Private: + this is the entry point wired into the lifecycle trigger lists + (the second entry in both ``AsgiLifespan`` and + ``AsgiRunOnFirstRequest``'s ``on_startup``, after + ``_startup_sequence``) - not something plugins or embedders should + call directly; use ``add_background_task`` / + ``start_background_tasks`` instead. + + Positioned after ``_startup_sequence`` in both trigger lists so + launch always happens once every plugin's ``startup`` hook has had + a chance to register work - the ordering guarantee that makes + ``add_background_task`` useful. No-ops when + ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its + one-shot TestClient request flows through the full ASGI stack, + including the first-request fallback, but must never launch + long-lived background work). + """ + if self._suppress_background_tasks: + return + await self._background_tasks.launch_all() + + async def invoke_shutdown(self): + """Run the graceful teardown sequence: plugin ``shutdown`` hooks, + then cancel and drain supervised background tasks, then close + every database. + """ + if self._shutdown_invoked: + return + self._shutdown_invoked = True + for hook in pm.hook.shutdown(datasette=self): + try: + await await_me_maybe(hook) + except Exception: + logging.getLogger("datasette").exception("shutdown hook failed") + await self._background_tasks.cancel_all(grace=5.0) + self.close() + def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() - async def setup_db(): - # First time server starts up, calculate table counts for immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - - async def _close_on_shutdown(): - self.close() - asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) + asgi = AsgiLifespan( + asgi, + on_startup=[self._startup_sequence, self._launch_background_tasks], + on_shutdown=[self.invoke_shutdown], + ) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) + asgi = AsgiRunOnFirstRequest( + asgi, + on_startup=[self._startup_sequence, self._launch_background_tasks], + ) + # Outermost, so spans from plugin middleware and first-request + # startup are children of the request span + asgi = TelemetryMiddleware(asgi) return asgi @@ -2860,6 +3207,50 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) + match, view = resolve_routes(self.routes, path) + is_static = view is favicon or getattr(view, "_datasette_static", False) + original_send = send + + async def send(message): + if message["type"] == "http.response.start" and not ( + is_static and message["status"] in (200, 304) + ): + # Decide privacy after rendering, including for streaming responses + # and error handlers. A public primary resource can still include + # private labels, actor navigation, or cookie-dependent content. + headers = list(message.get("headers", [])) + personalized = ( + request.actor is not None + or "cookie" in request.headers + or "authorization" in request.headers + or any(key.lower() == b"set-cookie" for key, _ in headers) + ) + if personalized: + headers = [ + (key, value) + for key, value in headers + if key.lower() != b"cache-control" + ] + headers.append((b"cache-control", b"private, no-store")) + + # Anonymous responses must not be reused for credentialed requests. + # Preserve any additional variation specified by views or plugins. + vary = [ + part.strip() + for key, value in headers + if key.lower() == b"vary" + for part in value.split(b",") + if part.strip() + ] + if b"*" not in vary: + for name in (b"Cookie", b"Authorization"): + if name.lower() not in {part.lower() for part in vary}: + vary.append(name) + headers = [(k, v) for k, v in headers if k.lower() != b"vary"] + headers.append((b"vary", b", ".join(vary))) + message = dict(message, headers=headers) + await original_send(message) + # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -2899,12 +3290,18 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - - match, view = resolve_routes(self.routes, path) + request.scope = scope if match is None: return await self.handle_404(request, send) + # Now the route is known, add it to the request span + span = request_span(scope) + if span is not None: + route = match.re.pattern + span.set_attribute(HTTP_ROUTE, route) + span.update_name(f"{clamp_http_method(request.method)} {route}") + new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) request.scope = new_scope try: @@ -3215,14 +3612,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3269,10 +3666,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx + **kwargs: Additional arguments to pass to httpx2 Returns: - httpx.Response: The response from the request + httpx2.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3281,16 +3678,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/background_tasks.py b/datasette/background_tasks.py new file mode 100644 index 00000000..6b34fd85 --- /dev/null +++ b/datasette/background_tasks.py @@ -0,0 +1,227 @@ +""" +Supervised background-task registration for Datasette core. + +Plugins that need long-lived background work (a polling loop, a queue +consumer, a scheduled job runner) register it with +``datasette.add_background_task(func, name=None)`` - typically from a +``startup`` plugin hook - instead of fire-and-forgetting their own +``asyncio.create_task()``. Core owns: + +- **references**: every launched ``asyncio.Task`` is kept alive on a + :class:`BackgroundTaskSupervisor`, so it can never be silently garbage + collected the way an unreferenced ``create_task()`` call can be; +- **launch timing**: registered work is buffered until + :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to + happen only after *every* plugin's ``startup`` hook has finished - so + a task that depends on another plugin having registered something first + doesn't need ``tryfirst=True`` ordering tricks; +- **crash surfacing**: an unhandled exception in a background task is + logged with its full traceback to the ``datasette.background_tasks`` + logger and recorded on the handle, instead of becoming an "Task + exception was never retrieved" warning nobody sees; +- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels + every task still running and waits (with a grace period) for them to + actually stop. +""" + +from __future__ import annotations + +import asyncio +import datetime +import functools +import logging +from collections.abc import Awaitable, Callable + +logger = logging.getLogger("datasette.background_tasks") + + +def _utcnow_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _function_path(func: Callable) -> str: + """Describe the callable without guessing which plugin registered it.""" + while isinstance(func, functools.partial): + func = func.func + if not hasattr(func, "__qualname__"): + func = type(func).__call__ + return f"{func.__module__}.{func.__qualname__}" + + +class BackgroundTask: + """A handle to a single piece of supervised background work. + + States: ``registered`` (added but not yet launched) -> ``running`` -> + one of ``completed`` (returned cleanly), ``crashed`` (raised an + exception other than ``CancelledError`` - see ``.exception``), or + ``cancelled`` (``.cancel()`` was called, or it was still running at + shutdown). + """ + + def __init__( + self, + name: str, + func: Callable[[object], Awaitable[None]], + ): + self.name = name + self.state = "registered" + self.task: asyncio.Task | None = None + self.exception: BaseException | None = None + self.started_at: str | None = None + self.function = _function_path(func) + self._func = func + self._supervisor: BackgroundTaskSupervisor | None = None + + def cancel(self) -> None: + """Cancel this task. + + If it has already been launched, cancels the underlying + ``asyncio.Task`` - its state becomes ``cancelled`` once the + cancellation is observed (asynchronously, via the task's done + callback). If it has not been launched yet, this is a no-op as + far as asyncio is concerned (there's no task to cancel) but it + deregisters the handle from its supervisor so it never runs. + """ + if self.task is not None: + self.task.cancel() + elif self._supervisor is not None: + self._supervisor._deregister(self) + + def __repr__(self) -> str: + return f"" + + +class BackgroundTaskSupervisor: + """Owns registration and launch of every :class:`BackgroundTask` for a + single ``Datasette`` instance. + + Registration (:meth:`add`) is separate from launch + (:meth:`launch_all`): plugins register work whenever convenient + (typically from a ``startup`` hook, but request handlers can register + dynamic per-job work too), and it either sits buffered until + :meth:`launch_all` runs, or - if :meth:`launch_all` has already run - + starts immediately. + + Strong references to every :class:`BackgroundTask` (and its + ``asyncio.Task``) are kept for the life of the instance, by design - + that's what makes the enrichments-style "fire-and-forget task gets + garbage collected mid-flight" bug impossible here. There is currently + no pruning of completed/crashed/cancelled tasks, so a plugin that + dynamically registers many short-lived tasks over a long process + lifetime (a per-job registration pattern, e.g. one task per queued + job) will grow this list without bound. That's an accepted v1 + trade-off in favour of full introspection (``/-/tasks``); revisit + with a pruning or capping policy if unbounded growth is reported in + practice. + """ + + def __init__(self, datasette): + self._datasette = datasette + self._tasks: list[BackgroundTask] = [] + self._names = set() + self._launched = False + self._lock = asyncio.Lock() + + def add(self, func, name=None) -> BackgroundTask: + base_name = name or getattr(func, "__qualname__", None) or repr(func) + actual_name = self._unique_name(base_name) + handle = BackgroundTask(actual_name, func) + handle._supervisor = self + self._tasks.append(handle) + self._names.add(actual_name) + if self._launched: + self._launch_one(handle) + return handle + + def _unique_name(self, base_name: str) -> str: + if base_name not in self._names: + return base_name + n = 2 + while f"{base_name}-{n}" in self._names: + n += 1 + return f"{base_name}-{n}" + + def _deregister(self, handle: BackgroundTask) -> None: + try: + self._tasks.remove(handle) + except ValueError: + pass + self._names.discard(handle.name) + + def _launch_one(self, handle: BackgroundTask) -> None: + handle.state = "running" + handle.started_at = _utcnow_iso() + handle.task = asyncio.create_task( + handle._func(self._datasette), name=handle.name + ) + handle.task.add_done_callback(functools.partial(_on_task_done, handle)) + + async def launch_all(self) -> None: + """Launch every currently-registered task that hasn't launched + yet. Idempotent and safe to call concurrently: subsequent (or + racing) calls are no-ops once the first has set ``self._launched``. + """ + if self._launched: + return + async with self._lock: + if self._launched: + return + self._launched = True + for handle in list(self._tasks): + if handle.task is None: + self._launch_one(handle) + + async def cancel_all(self, grace: float = 5.0) -> None: + """Cancel every task that isn't already done, then wait up to + ``grace`` seconds for them to actually finish. Stragglers still + running after that are logged by name (but left to finish or not + on their own - this does not forcibly kill them, asyncio has no + mechanism for that). + """ + handles_by_task = { + handle.task: handle for handle in self._tasks if handle.task is not None + } + pending = [task for task in handles_by_task if not task.done()] + for task in pending: + task.cancel() + if not pending: + return + _done, not_done = await asyncio.wait(pending, timeout=grace) + if not_done: + names = sorted(handles_by_task[task].name for task in not_done) + logger.warning( + "%d background task(s) did not finish within the %.1fs grace " + "period after cancellation: %s", + len(names), + grace, + ", ".join(names), + ) + + def tasks(self) -> list[BackgroundTask]: + """Return every registered :class:`BackgroundTask`, launched or + not, in registration order. Used by the ``/-/tasks`` debug + endpoint. + """ + return list(self._tasks) + + @property + def launched(self) -> bool: + """Whether :meth:`launch_all` has run yet - lets ``/-/tasks`` + distinguish "no tasks registered" from "tasks registered but + nothing has armed the launch yet" without reaching for the + private ``_launched`` attribute. + """ + return self._launched + + +def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None: + if task.cancelled(): + handle.state = "cancelled" + return + exc = task.exception() + if exc is not None: + handle.state = "crashed" + handle.exception = exc + logger.error("Background task %r crashed", handle.name, exc_info=exc) + return + handle.state = "completed" diff --git a/datasette/cli.py b/datasette/cli.py index 57db83b6..e83de93a 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -157,7 +157,11 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) + + def _inspect_tables(conn): + return inspect_tables(conn, {}) + + tables = await database.execute_fn(_inspect_tables) data[name] = { "hash": database.hash, "size": database.size, @@ -497,6 +501,7 @@ def uninstall(packages, yes): "--internal", type=click.Path(), help="Path to a persistent Datasette internal SQLite database", + envvar="DATASETTE_INTERNAL", ) def serve( files, @@ -663,16 +668,6 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run async soundness checks before startup hooks, since invoke_startup - # now populates internal tables which requires querying each database - run_sync(lambda: check_databases(ds)) - - # Run the "startup" plugin hooks - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -680,6 +675,19 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: + # --get means we don't run Uvicorn at all + run_sync(lambda: check_databases(ds)) + + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + + # --get never launches background tasks: TestClient's request below + # flows through the full ASGI stack, including the + # AsgiRunOnFirstRequest fallback, which would otherwise launch them. + ds._suppress_background_tasks = True + client = TestClient(ds) request_headers = {} if token: @@ -704,34 +712,54 @@ def serve( sys.exit(exit_code) return - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = run_sync(lambda: initial_path_for_datasette(ds)) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - uvicorn.run(ds.app(), **uvicorn_kwargs) + # check_databases, invoke_startup() and the uvicorn server all run on a + # single event loop, so that anything a plugin's "startup" hook schedules + # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is + # still alive when the server starts handling requests. + async def _serve_async(): + # Populate internal catalog tables before invoke_startup + await check_databases(ds) + + # Run the full startup sequence (immutable-database table-count + # precompute + the "startup" plugin hooks) via the same entry point + # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when + # uvicorn's lifespan.startup fires moments later. + try: + await ds._startup_sequence() + except StartupError as e: + raise click.ClickException(e.args[0]) + + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = await initial_path_for_datasette(ds) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) + await server.serve() + + asyncio.run(_serve_async()) @cli.command() diff --git a/datasette/database.py b/datasette/database.py index e162d34e..542b3012 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,18 +1,54 @@ import asyncio import atexit +import contextvars import inspect import os import queue import sys import tempfile import threading +import time import uuid from collections import namedtuple from pathlib import Path import sqlite_utils +from opentelemetry import context as otel_context_api +from opentelemetry.trace import Status, StatusCode from .inspect import inspect_hash +from .telemetry import ( + callback_name, + linked_root_span_kwargs, + record_operation_duration, + record_query_interrupted, + record_write_queue_wait, + sql_attribute, + sql_operation_name, + tracer, +) +from .telemetry_registry import ( + CALLBACK, + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY, + DB_QUERY_EXECUTE, + DB_QUERY_TEXT, + DB_SYSTEM, + DB_WRITE_EXECUTE, + DB_WRITE_QUEUE_WAIT, + EXECUTEMANY, + EXECUTESCRIPT, + INTERRUPTED, + ISOLATED_CONNECTION, + PARAM_COUNT, + PARAM_SETS, + ROWS_RETURNED, + SQL_ERROR_SUPPRESSED, + TIME_LIMIT_MS, + TRANSACTION, + TRUNCATED, +) from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -29,7 +65,7 @@ from .utils import ( table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_hidden_table_names +from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names connections = threading.local() @@ -85,6 +121,7 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None + self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -93,8 +130,9 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - # This is used to track all file connections so they can be closed - self._all_file_connections = [] + # Track file and memory connections, including reads on worker threads, + # so close() can release all of them from the calling thread. + self._all_connections = [] if not is_temp_disk: self.mode = mode @@ -145,9 +183,12 @@ class Database: ) if not write: conn.execute("PRAGMA query_only=1") + self._all_connections.append(conn) return conn if self.is_memory: - return sqlite3.connect(":memory:", uri=True) + conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False) + self._all_connections.append(conn) + return conn # mode=ro or immutable=1? if self.is_mutable: @@ -164,7 +205,7 @@ class Database: conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) - self._all_file_connections.append(conn) + self._all_connections.append(conn) if self.is_temp_disk and not self._wal_enabled: conn.execute("PRAGMA journal_mode=WAL") self._wal_enabled = True @@ -201,13 +242,13 @@ class Database: except Exception: # noqa: BLE001, S110 # Shutdown teardown - a failed pending write must not block close() pass - # Close anything still tracked in _all_file_connections - for connection in self._all_file_connections: + # Close anything still tracked in _all_connections + for connection in self._all_connections: try: connection.close() except Exception: # noqa: BLE001, S110 pass - self._all_file_connections = [] + self._all_connections = [] # Drop per-thread cached read connections we can reach try: delattr(connections, self._thread_local_id) @@ -246,21 +287,45 @@ class Database: return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, transaction=True, + time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def _inner(conn): + def execute_sql(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn( - _inner, block=block, request=request, transaction=transaction - ) + def _inner(conn): + try: + if time_limit_ms is None: + return execute_sql(conn) + with sqlite_timelimit(conn, time_limit_ms): + return execute_sql(conn) + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + raise + + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), params=params + ): + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute(DB_OPERATION_NAME, operation_name) + if params: + span.set_attribute(PARAM_COUNT, len(params)) + with record_operation_duration(self.name, "write"): + results = await self._execute_write_fn( + _inner, block=block, request=request, transaction=transaction + ) return results async def execute_write_script(self, sql, block=True, request=None): @@ -269,10 +334,19 @@ class Database: def _inner(conn): return conn.executescript(sql) - with trace("sql", database=self.name, sql=sql.strip(), executescript=True): - results = await self.execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), executescript=True + ): + # No db.operation.name, since the script can contain multiple statements + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(EXECUTESCRIPT, True) + with record_operation_duration(self.name, "write"): + results = await self._execute_write_fn( + _inner, block=block, transaction=False, request=request + ) return results async def execute_write_many(self, sql, params_seq, block=True, request=None): @@ -292,9 +366,19 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - results, count = await self.execute_write_fn( - _inner, block=block, request=request - ) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(EXECUTEMANY, True) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute(DB_OPERATION_NAME, operation_name) + with record_operation_duration(self.name, "write"): + results, count = await self._execute_write_fn( + _inner, block=block, request=request + ) + span.set_attribute(PARAM_SETS, count) kwargs["count"] = count return results @@ -311,31 +395,58 @@ class Database: finally: isolated_connection.close() try: - self._all_file_connections.remove(isolated_connection) + self._all_connections.remove(isolated_connection) except ValueError: - # Was probably a memory connection + # May already have been cleared by close(). pass - if self.ds.executor is None: - # non-threaded mode - return _run() - if not write: - # Immutable database - no writes can ever occur, so there is no - # write queue to block; run against a fresh read-only connection - return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, _run - ) - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, callback_name(fn)) + # Immutable databases run this on the read pool, not the write queue + with record_operation_duration(self.name, "write" if write else "read"): + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there + # is no write queue to block; run against a fresh + # read-only connection + ctx = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, ctx.run, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() - return await self.execute_isolated_fn( - lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) - ) + def _analyze_sql(conn): + return analyze_sql_tables(conn, sql, params, database_name=self.name) + + return await self.execute_isolated_fn(_analyze_sql) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): + """Run `fn(conn)` on the write connection, traced as a `db.query` span. + + The SQL-string write methods call `_execute_write_fn()` directly to + avoid creating a second span. + """ + self._check_not_closed() + # Record the name before _wrap_fn_with_hooks() wraps fn + name = callback_name(fn) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, name) + with record_operation_duration(self.name, "write"): + return await self._execute_write_fn( + fn, block=block, transaction=transaction, request=request + ) + + async def _execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() pending_events = [] @@ -354,6 +465,15 @@ class Database: result = fn(self._write_connection) else: result = fn(self._write_connection) + if not block: + # There is no write thread here, so the write has already + # finished. Hand back the same (task_id, reply_future) shape + # _send_to_write_thread() returns, with the future already + # resolved, so the block=False path below is identical in + # both modes. + reply_future = asyncio.get_running_loop().create_future() + reply_future.set_result(result) + result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -425,11 +545,22 @@ class Database: ) self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + task_id = uuid.uuid4() loop = asyncio.get_running_loop() reply_future = loop.create_future() + # Capture the OpenTelemetry context and enqueue time for the write thread self._write_queue.put( - WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) + WriteTask( + fn, + task_id, + loop, + reply_future, + isolated_connection, + transaction, + otel_context_api.get_current(), + time.time_ns(), + block, + ) ) if block: return await reply_future @@ -443,6 +574,8 @@ class Database: conn = None try: conn = self.connect(write=True) + # Threads do not inherit the caller's context, so any spans + # created by prepare_connection hooks here are root spans self.ds._prepare_connection(conn, self.name) except Exception as e: # noqa: BLE001 # Stored and re-raised to whoever queues the next write @@ -457,42 +590,101 @@ class Database: # Best-effort close as the write thread exits pass return - exception = None - result = None - if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - try: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e + # block=True: the caller awaits the result, so the write spans + # are children of the caller's span. The token must be detached + # in the finally block or the context leaks into later writes. + # block=False: the caller may finish first, so the write spans + # are root spans with a link back to the caller's span. + token = None + write_span_kwargs = {} + if task.block: + token = otel_context_api.attach(task.otel_context) else: - try: - if task.transaction: - with conn: - conn.execute("BEGIN IMMEDIATE") - result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) + write_span_kwargs = linked_root_span_kwargs(task.otel_context) + try: + exception = None + result = None + # Span covers the time from enqueue to dequeue + dequeued_at_ns = time.time_ns() + tracer.start_span( + DB_WRITE_QUEUE_WAIT, + start_time=task.enqueued_at_ns, + **write_span_kwargs, + ).end(end_time=dequeued_at_ns) + record_write_queue_wait(self.name, dequeued_at_ns - task.enqueued_at_ns) + if conn_exception is not None: + exception = conn_exception + elif task.isolated_connection: + try: + with tracer.start_as_current_span( + DB_WRITE_EXECUTE, **write_span_kwargs + ) as span: + span.set_attribute( + ISOLATED_CONNECTION, + task.isolated_connection, + ) + span.set_attribute(TRANSACTION, task.transaction) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_connections.remove(isolated_connection) + except ValueError: + # May already have been cleared by close(). + pass + except Exception as e: # noqa: BLE001 + # Write thread must survive any task failure or the database wedges + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + else: + try: + with tracer.start_as_current_span( + DB_WRITE_EXECUTE, **write_span_kwargs + ) as span: + span.set_attribute( + ISOLATED_CONNECTION, + task.isolated_connection, + ) + span.set_attribute(TRANSACTION, task.transaction) + if task.transaction: + with conn: + conn.execute("BEGIN IMMEDIATE") + result = task.fn(conn) + else: + result = task.fn(conn) + except Exception as e: # noqa: BLE001 + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) + finally: + if token is not None: + otel_context_api.detach(token) async def execute_fn(self, fn): + """Run `fn(conn)` on a read connection, traced as a `db.query` span. + + `execute()` calls `_execute_fn()` directly to avoid creating a second + span. + """ + self._check_not_closed() + + def fn_in_execute_span(conn): + # Runs on the worker thread + with tracer.start_as_current_span(DB_QUERY_EXECUTE): + return fn(conn) + + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, callback_name(fn)) + with record_operation_duration(self.name, "read"): + return await self._execute_fn(fn_in_execute_span) + + async def _execute_fn(self, fn): self._check_not_closed() if self.ds.executor is None: # non-threaded mode @@ -512,7 +704,11 @@ class Database: with self._pending_execute_futures_lock: self._check_not_closed() - future = self.ds.executor.submit(in_thread) + # Run in a copy of the caller's context so spans created in the + # thread have the correct parent. This needs a fresh copy for + # each submit, since a Context cannot be entered concurrently. + ctx = contextvars.copy_context() + future = self.ds.executor.submit(ctx.run, in_thread) self._pending_execute_futures.add(future) future.add_done_callback(self._remove_pending_execute_future) return await asyncio.wrap_future(future) @@ -529,44 +725,101 @@ class Database: """Executes sql against db_name in a thread""" self._check_not_closed() page_size = page_size or self.ds.page_size + time_limit_ms = self.ds.sql_time_limit_ms + # Callers that pass a shorter custom_time_limit, such as table counts + # and facet suggestions, expect timeouts, so they are not span errors + timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms + if timeout_expected: + time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): - time_limit_ms = self.ds.sql_time_limit_ms - if custom_time_limit and custom_time_limit < time_limit_ms: - time_limit_ms = custom_time_limit - - with sqlite_timelimit(conn, time_limit_ms): + # Expected timeouts and errors with log_sql_errors=False are not + # recorded as span errors, so exceptions are handled explicitly + with tracer.start_as_current_span( + DB_QUERY_EXECUTE, + record_exception=False, + set_status_on_exception=False, + ) as execute_span: try: - cursor = conn.cursor() - cursor.execute(sql, params if params is not None else {}) - max_returned_rows = self.ds.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params if params is not None else {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + sys.stderr.write( + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + ) + sys.stderr.flush() + raise + except QueryInterrupted as e: + if not timeout_expected: + execute_span.record_exception(e) + execute_span.set_status(Status(StatusCode.ERROR, str(e))) + raise + except Exception as e: if log_sql_errors: - sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" - ) - sys.stderr.flush() + execute_span.record_exception(e) + execute_span.set_status(Status(StatusCode.ERROR, str(e))) raise - if truncate: - return Results(rows, truncated, cursor.description) + if truncate: + return Results(rows, truncated, cursor.description) - else: - return Results(rows, False, cursor.description) + else: + return Results(rows, False, cursor.description) - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_fn(sql_operation_in_thread) + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), params=params + ): + with tracer.start_as_current_span( + DB_QUERY, + kind=DB_QUERY.kind, + record_exception=False, + set_status_on_exception=False, + ) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(TIME_LIMIT_MS, time_limit_ms) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute(DB_OPERATION_NAME, operation_name) + if params: + span.set_attribute(PARAM_COUNT, len(params)) + try: + with record_operation_duration(self.name, "read"): + results = await self._execute_fn(sql_operation_in_thread) + except QueryInterrupted as e: + span.set_attribute(INTERRUPTED, True) + if not timeout_expected: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) + record_query_interrupted(self.name) + raise + except Exception as e: + # log_sql_errors=False callers, such as facet suggestion, + # expect some queries to fail + if log_sql_errors: + span.record_exception(e) + span.set_status(Status(StatusCode.ERROR, str(e))) + else: + span.set_attribute(SQL_ERROR_SUPPRESSED, True) + raise + span.set_attribute(TRUNCATED, results.truncated) + span.set_attribute(ROWS_RETURNED, len(results.rows)) return results @property @@ -657,17 +910,32 @@ class Database: ) return [r[0] for r in results.rows] + # Named functions rather than lambdas give more useful datasette.callback + # span attributes + async def table_columns(self, table): - return await self.execute_fn(lambda conn: table_columns(conn, table)) + def _table_columns(conn): + return table_columns(conn, table) + + return await self.execute_fn(_table_columns) async def table_column_details(self, table): - return await self.execute_fn(lambda conn: table_column_details(conn, table)) + def _table_column_details(conn): + return table_column_details(conn, table) + + return await self.execute_fn(_table_column_details) async def primary_keys(self, table): - return await self.execute_fn(lambda conn: detect_primary_keys(conn, table)) + def _primary_keys(conn): + return detect_primary_keys(conn, table) + + return await self.execute_fn(_primary_keys) async def fts_table(self, table): - return await self.execute_fn(lambda conn: detect_fts(conn, table)) + def _fts_table(conn): + return detect_fts(conn, table) + + return await self.execute_fn(_fts_table) async def label_column_for_table(self, table): explicit_label_column = (await self.ds.table_config(self.name, table)).get( @@ -759,6 +1027,17 @@ class Database: return hidden_tables + async def derived_table_dependencies(self): + """Return implementation tables and the tables they derive from.""" + schema_version = (await self.execute("PRAGMA schema_version")).first()[0] + if ( + self._cached_derived_table_dependencies is None + or self._cached_derived_table_dependencies[0] != schema_version + ): + dependencies = await self.execute_fn(sqlite_derived_table_dependencies) + self._cached_derived_table_dependencies = (schema_version, dependencies) + return self._cached_derived_table_dependencies[1] + async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] @@ -854,16 +1133,28 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( + "block", + "enqueued_at_ns", "fn", "isolated_connection", "loop", + "otel_context", "reply_future", "task_id", "transaction", ) def __init__( - self, fn, task_id, loop, reply_future, isolated_connection, transaction + self, + fn, + task_id, + loop, + reply_future, + isolated_connection, + transaction, + otel_context, + enqueued_at_ns, + block, ): self.fn = fn self.task_id = task_id @@ -871,6 +1162,9 @@ class WriteTask: self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction + self.otel_context = otel_context + self.enqueued_at_ns = enqueued_at_ns + self.block = block def _deliver_write_result(task, result, exception): diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index f90a733e..6def3698 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,6 +6,17 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType +_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) + + +def _normalize_http_url(value): + if not isinstance(value, str): + return None + normalized = value.strip() + if not _HTTP_URL_RE.fullmatch(normalized): + return None + return normalized + class UrlColumnType(ColumnType): name = "url" @@ -15,7 +26,10 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - escaped = markupsafe.escape(value.strip()) + normalized = _normalize_http_url(value) + if normalized is None: + return markupsafe.escape(value.strip()) + escaped = markupsafe.escape(normalized) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -23,7 +37,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if not re.match(r"^https?://\S+$", value.strip()): + if _normalize_http_url(value) is None: return "Invalid URL" return None diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index 4494f07f..a4f5a4de 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -92,6 +92,13 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) + # Resolve identity keys once per action, rather than scanning the + # restriction allowlist for every configured table's allow block. + self.restricted_table_keys = { + (db, self.action_obj.normalize_child(table) if self.action_obj else table) + for db, table in self.restricted_tables + } + def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: @@ -125,8 +132,10 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - table_actions = table_restrictions.get(child, []) - if self.action_checks.intersection(table_actions): + child_key = ( + self.action_obj.normalize_child(child) if self.action_obj else child + ) + if (parent, child_key) in self.restricted_table_keys: return True else: # Parent query should proceed if any child in this database is allowlisted diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index 88e1d274..d30ebd3f 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -185,11 +185,15 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + action_obj = datasette.actions.get(action) + normalize = action_obj.normalize_child if action_obj else lambda name: name + for table_name, table_allowed in ( + restrictions.get("r", {}).get(database, {}).items() + ): + if normalize(table_name) == normalize(table): + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py new file mode 100644 index 00000000..11fd4008 --- /dev/null +++ b/datasette/default_permissions/sqlite_statistics.py @@ -0,0 +1,25 @@ +"""Default table-access policy for SQLite optimizer statistics.""" + +import json + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + + +@hookimpl +def permission_resources_sql(action): + if action != "view-table": + return None + return PermissionSQL( + sql=""" + SELECT database_name AS parent, value AS child, 0 AS allow, + 'SQLite statistics tables are denied by default' AS reason + FROM catalog_databases + CROSS JOIN json_each(:sqlite_statistics_names) + """, + params={ + "sqlite_statistics_names": json.dumps( + ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] + ) + }, + ) diff --git a/datasette/facets.py b/datasette/facets.py index 8c09e1dc..69ac2c42 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -39,7 +39,7 @@ def load_facet_configs(request, table_config): ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): - if key.startswith("_facet"): + if key == "_facet" or key.startswith("_facet_"): # Figure out the facet type if key == "_facet": type = "column" @@ -264,10 +264,15 @@ class ColumnFacet(Facet): column_qs = column if column.startswith("_"): column_qs = f"{column}__exact" - selected = (column_qs, str(row["value"])) in qs_pairs + selected_args = { + key: str(row["value"]) + for key in (column_qs, f"{column}__exact") + if (key, str(row["value"])) in qs_pairs + } + selected = bool(selected_args) if selected: toggle_path = path_with_removed_args( - self.request, {column_qs: str(row["value"])} + self.request, selected_args ) else: toggle_path = path_with_added_args( diff --git a/datasette/filters.py b/datasette/filters.py index 3cfb36e5..0499d086 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,8 +1,9 @@ import json +import math from typing import ClassVar from datasette import hookimpl -from datasette.resources import DatabaseResource +from datasette.resources import DatabaseResource, TableResource from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError @@ -51,13 +52,20 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which fts_table to use + # Figure out which trusted fts_table to use. Query string parameters can + # repeat this mapping (for backwards compatibility), but must not select + # a different table or primary key. table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = request.args.get("_fts_table") - fts_table = fts_table or table_metadata.get("fts_table") + fts_table = table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) + fts_pk = table_metadata.get("fts_pk", "rowid") + requested_fts_table = request.args.get("_fts_table") + requested_fts_pk = request.args.get("_fts_pk") + if (requested_fts_table and requested_fts_table != fts_table) or ( + requested_fts_pk and requested_fts_pk != fts_pk + ): + raise BadRequest("Invalid _fts_table or _fts_pk") search_args = { key: request.args[key] for key in request.args @@ -75,6 +83,11 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=fts_table), + actor=request.actor, + ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -135,6 +148,11 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=through_table), + actor=request.actor, + ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) fk_to_us = next( @@ -185,6 +203,17 @@ class Filter: raise NotImplementedError +def _coerce_numeric_filter_value(value): + try: + return int(value) + except ValueError: + try: + converted = float(value) + except ValueError: + return value + return converted if math.isfinite(converted) else value + + class TemplatedFilter(Filter): def __init__( self, @@ -206,8 +235,8 @@ class TemplatedFilter(Filter): def where_clause(self, table, column, value, param_counter): converted = self.format.format(value) - if self.numeric and converted.isdigit(): - converted = int(converted) + if self.numeric: + converted = _coerce_numeric_filter_value(converted) if self.no_argument: kwargs = {"c": _quote_sqlite_identifier(column)} converted = None diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index c36d5dbe..ef6c7b7e 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -59,6 +59,10 @@ def handle_exception(datasette, request, exception): body = dict(info) body.update(error_body(plain_message or message, status)) return Response.json(body, status=status, headers=headers) + if request.path.split("?")[0].endswith(".csv"): + return Response.text( + plain_message or message, status=status, headers=headers + ) info.update( { "ok": False, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index f89f2f36..49d8e8ea 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -9,6 +9,11 @@ def startup(datasette): """Fires directly after Datasette first starts running""" +@hookspec +def shutdown(datasette): + """Called once when the Datasette server is shutting down""" + + @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" @@ -45,7 +50,7 @@ def extra_body_script( def extra_template_vars( template, database, table, columns, view_name, request, datasette ): - """Extra template variables to be made available to the template - can return dict or callable or awaitable""" + """Extra template variables to be made available to the template - can return dict, None, callable or awaitable""" @hookspec diff --git a/datasette/permissions.py b/datasette/permissions.py index e03b065c..2d242560 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -3,6 +3,10 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple +_SQLITE_IDENTIFIER_CASE = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) + # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False @@ -49,6 +53,15 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables + case_insensitive_child: bool = False + + @classmethod + def normalize_child(cls, child: str | None) -> str | None: + """Return a comparison key without changing the resource's display name.""" + if cls.case_insensitive_child and child is not None: + # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. + return child.translate(_SQLITE_IDENTIFIER_CASE) + return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -146,6 +159,11 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed + def normalize_child(self, child: str | None) -> str | None: + if self.resource_class is None: + return child + return self.resource_class.normalize_child(child) + @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 9cf94079..6a4d7da7 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -18,6 +18,7 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", + "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", diff --git a/datasette/resources.py b/datasette/resources.py index ee2e6d98..29bf7b1e 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,6 +25,7 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource + case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/app.css b/datasette/static/app.css index d101e4b7..0297371c 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1,3 +1,144 @@ +/* Shared modal styles. */ +datasette-modal { + display: contents; +} + +dialog.datasette-modal { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(520px, calc(100vw - 32px)); + max-width: 95vw; + max-height: calc(100dvh - 32px); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.datasette-modal[open] { + display: flex; + flex-direction: column; +} + +dialog.datasette-modal::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +@keyframes datasette-modal-slide-in { + from { opacity: 0; transform: translateY(-20px) scale(0.95); } + to { opacity: 1; transform: translateY(0) scale(1); } +} + +@keyframes datasette-modal-fade-in { + from { opacity: 0; } + to { opacity: 1; } +} + +:where(.datasette-modal) .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; +} + +:where(.datasette-modal) .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +:where(.datasette-modal) .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; +} + +:where(.datasette-modal) .modal-body { + min-height: 0; + overflow: auto; + padding: 16px 24px 24px; +} + +:where(.datasette-modal) .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +:where(.datasette-modal) .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +:where(.datasette-modal) .modal-btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +:where(.datasette-modal) .modal-btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +:where(.datasette-modal) .modal-btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +:where(.datasette-modal) .modal-btn-primary { + background: var(--accent); + color: #fff; +} + +:where(.datasette-modal) .modal-btn-primary:hover { + background: #1949b8; +} + +:where(.datasette-modal) .modal-btn:disabled { + opacity: 0.65; + cursor: wait; +} + +@media (prefers-reduced-motion: reduce) { + dialog.datasette-modal, + dialog.datasette-modal::backdrop { + animation: none; + } +} + /* Reset and Page Setup ==================================================== */ /* Reset from http://meyerweb.com/eric/tools/css/reset/ @@ -63,7 +204,7 @@ em { } /* end reset */ -/* Modal CSS variables (shared by web components via Shadow DOM) */ +/* Shared modal CSS variables */ :root { --modal-backdrop-bg: rgba(0, 0, 0, 0.5); --modal-backdrop-blur: blur(4px); @@ -216,6 +357,49 @@ a:active { text-decoration: underline; } +.table-summary .count-all ~ .table-summary-description { + margin-left: 0.5rem; +} + +.table-summary .count-error:not(:empty) { + display: block; + margin-top: 0.25rem; + font-size: 0.875rem; + font-weight: 400; + line-height: 1.5; +} + +button.count-all { + background: none; + border: none; + padding: 3px 0; + margin-left: 0.25rem; + color: #276890; + font-family: inherit; + font-size: 0.8125rem; + font-weight: 400; + line-height: 1.5; + cursor: pointer; +} + +button.count-all:hover, +button.count-all:focus-visible { + text-decoration: underline; +} + +button.count-all:disabled { + color: #596478; + cursor: wait; +} + +@media (pointer: coarse) { + button.count-all { + min-height: 44px; + padding-left: 7px; + padding-right: 7px; + } +} + button.button-as-link { background: none; border: none; @@ -938,84 +1122,552 @@ p.zero-results { display: none; } -@keyframes datasette-modal-slide-in { - from { - opacity: 0; - transform: translateY(-20px) scale(0.95); +/* navigation-search */ +navigation-search { + display: contents; +} + +navigation-search dialog.datasette-modal { + max-width: 90vw; + width: 600px; + max-height: 80vh; +} + +navigation-search .search-container { + display: flex; + flex-direction: column; +} + +navigation-search .search-input-wrapper { + padding: 1.25rem; + border-bottom: 1px solid #e5e7eb; + display: flex; + gap: 0.5rem; + align-items: center; +} + +navigation-search .search-input { + width: 100%; + flex: 1; + min-width: 0; + padding: 0.75rem 1rem; + font-size: 1rem; + border: 2px solid #e5e7eb; + border-radius: 0.5rem; + outline: none; + transition: border-color 0.2s; + box-sizing: border-box; +} + +navigation-search .search-input:focus { + border-color: #2563eb; +} + +navigation-search .close-search { + background: transparent; + border: 1px solid transparent; + border-radius: 0.375rem; + color: #4b5563; + cursor: pointer; + flex: 0 0 auto; + font: inherit; + font-size: 1.5rem; + height: 2.75rem; + line-height: 1; + width: 2.75rem; +} + +navigation-search .close-search:hover, +navigation-search .close-search:focus { + background-color: #f3f4f6; + border-color: #d1d5db; +} + +navigation-search .results-container { + box-sizing: content-box; + height: calc(80vh - 180px); + padding: 0.5rem; +} + +navigation-search .results-list:empty { + display: none; +} + +navigation-search .result-item { + padding: 0.875rem 1rem; + cursor: pointer; + border-radius: 0.5rem; + transition: background-color 0.15s; + display: flex; + align-items: center; + gap: 0.75rem; +} + +navigation-search .result-item:hover { + background-color: #f3f4f6; +} + +navigation-search .result-item.selected { + background-color: #dbeafe; +} + +navigation-search .result-item > div { + flex: 1; + min-width: 0; +} + +navigation-search .jump-start-content { + border-bottom: 1px solid #e5e7eb; + margin-bottom: 0.5rem; + padding: 0.5rem 0.5rem 1rem; +} + +navigation-search .jump-start-content:empty { + display: none; +} + +navigation-search .result-name { + font-weight: 500; + color: #111827; +} + +navigation-search .result-label { + font-size: 0.875rem; + color: #4b5563; +} + +navigation-search .result-type { + color: #4b5563; + font-size: 0.75rem; + font-weight: 600; + text-transform: uppercase; +} + +navigation-search .result-url { + font-size: 0.875rem; + color: #6b7280; +} + +navigation-search .result-description { + color: #374151; + display: -webkit-box; + font-size: 0.8125rem; + line-height: 1.35; + margin-top: 0.35rem; + overflow: hidden; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; +} + +navigation-search .results-heading { + color: #4b5563; + font-size: 0.75rem; + font-weight: 600; + letter-spacing: 0; + padding: 0.5rem 1rem 0.25rem; + text-transform: uppercase; +} + +navigation-search .recent-actions { + padding: 0.25rem 1rem 0.75rem; +} + +navigation-search .clear-recent { + background: transparent; + border: 0; + color: #2563eb; + cursor: pointer; + font: inherit; + font-size: 0.875rem; + padding: 0; +} + +navigation-search .clear-recent:hover { + text-decoration: underline; +} + +navigation-search .no-results { + padding: 2rem; + text-align: center; + color: #6b7280; +} + +navigation-search .hint-text { + padding: 0.75rem 1.25rem; + font-size: 0.875rem; + color: #6b7280; + border-top: 1px solid #e5e7eb; + display: flex; + gap: 1rem; + flex-wrap: wrap; +} + +navigation-search .hint-text kbd { + background: #f3f4f6; + padding: 0.125rem 0.375rem; + border-radius: 0.25rem; + font-size: 0.75rem; + border: 1px solid #d1d5db; + font-family: monospace; +} + +navigation-search .visually-hidden { + border: 0; + clip: rect(0 0 0 0); + height: 1px; + margin: -1px; + overflow: hidden; + padding: 0; + position: absolute; + white-space: nowrap; + width: 1px; +} + +@media (max-width: 640px) { + navigation-search dialog.datasette-modal { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; } - to { - opacity: 1; - transform: translateY(0) scale(1); + + navigation-search .search-input-wrapper { + padding: 1rem; + } + + navigation-search .search-input { + font-size: 16px; + } + + navigation-search .result-item { + padding: 1rem 0.75rem; + } + + navigation-search .hint-text { + font-size: 0.8rem; + padding: 0.5rem 1rem; } } -@keyframes datasette-modal-fade-in { - from { opacity: 0; } - to { opacity: 1; } -} -dialog.mobile-column-actions-dialog { +/* column-chooser */ +column-chooser { + display: contents; --ink: #0f0f0f; --paper: #eef6ff; --muted: #6b6b6b; --rule: #d8e6f5; --accent: #1a56db; + --accent-light: #e8effd; --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); +} + +column-chooser * { + box-sizing: border-box; + margin: 0; padding: 0; - margin: auto; - width: min(420px, calc(100vw - 32px)); - max-width: 95vw; +} + +column-chooser dialog.datasette-modal { + width: 100%; + max-width: 420px; max-height: min(640px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); + -webkit-user-select: none; + -webkit-touch-callout: none; + -webkit-tap-highlight-color: transparent; } -dialog.mobile-column-actions-dialog[open] { - display: flex; - flex-direction: column; +column-chooser dialog.datasette-modal[open] { + height: min(640px, calc(100vh - 32px)); } -dialog.mobile-column-actions-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.mobile-column-actions-dialog .modal-header { +column-chooser .modal-header { padding: 20px 24px 16px; + justify-content: space-between; +} + +column-chooser .list-toolbar { + padding: 6px 24px; border-bottom: 1px solid var(--rule); display: flex; - align-items: center; - justify-content: space-between; gap: 12px; flex-shrink: 0; } -.mobile-column-actions-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); +column-chooser .list-toolbar button { + background: var(--accent-light); + border: 1px solid var(--rule); + border-radius: 4px; + font-family: inherit; + font-size: 0.75rem; + color: var(--accent); + cursor: pointer; + padding: 3px 10px; + transition: + background 0.12s, + color 0.12s; } -.mobile-column-actions-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; +column-chooser .list-toolbar button:hover { + background: var(--accent); + color: white; +} + +column-chooser .list-wrap { + flex: 1; + padding: 0; + overflow-x: hidden; + position: relative; + overscroll-behavior: contain; + -webkit-overflow-scrolling: touch; +} + +column-chooser .list-wrap::before, +column-chooser .list-wrap::after { + content: ""; + position: sticky; + display: block; + left: 0; + right: 0; + height: 20px; + pointer-events: none; + z-index: 5; + transition: opacity 0.2s; +} + +column-chooser .list-wrap::before { + top: 0; + background: linear-gradient( + to bottom, + rgba(255, 255, 255, 0.9), + transparent + ); +} + +column-chooser .list-wrap::after { + bottom: 0; + background: linear-gradient(to top, rgba(255, 255, 255, 0.9), transparent); + margin-top: -20px; +} + +column-chooser .scroll-zone { + position: absolute; + left: 0; + right: 0; + height: 72px; + pointer-events: none; + z-index: 10; +} + +column-chooser .scroll-zone-top { + top: 0; +} + +column-chooser .scroll-zone-bot { + bottom: 0; +} + +column-chooser .drag-list { + list-style: none; + padding: 4px 0; +} + +column-chooser .drag-item { + display: flex; + align-items: center; + background: white; + border-bottom: 1px solid var(--rule); + user-select: none; + -webkit-user-select: none; + -webkit-touch-callout: none; + position: relative; + transition: background 0.08s; +} + +column-chooser .drag-item:last-child { + border-bottom: none; +} + +column-chooser .drag-handle { + display: flex; + align-items: center; + justify-content: center; + width: 48px; + height: 48px; + flex-shrink: 0; + cursor: grab; + color: #c8c4bc; + touch-action: none; + transition: color 0.15s; +} + +column-chooser .drag-handle:hover { + color: var(--accent); +} + +column-chooser .drag-handle svg { + pointer-events: none; + display: block; +} + +column-chooser .drag-item-content { + display: flex; + align-items: center; + flex: 1; + min-width: 0; + cursor: pointer; +} + +column-chooser .drag-item-check { + display: flex; + align-items: center; + width: 32px; + height: 48px; + flex-shrink: 0; +} + +column-chooser .drag-item-check input[type="checkbox"] { + width: 16px; + height: 16px; + accent-color: var(--accent); + cursor: pointer; +} + +column-chooser .drag-item-label { + flex: 1; + font-size: 0.9rem; + line-height: 48px; + padding-right: 16px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + cursor: default; +} + +column-chooser .drag-item.is-dragging { + opacity: 0; +} + +column-chooser .drop-indicator { + position: absolute; + left: 48px; + right: 0; + height: 2px; + background: var(--accent); + border-radius: 99px; + pointer-events: none; + z-index: 20; + display: none; +} + +column-chooser .drop-indicator.top { + top: -1px; + display: block; +} + +column-chooser .drop-indicator.bottom { + bottom: -1px; + display: block; +} + +column-chooser .drag-ghost { + position: fixed; + pointer-events: none; + z-index: 9999; + background: white; + border-radius: 6px; + box-shadow: + 0 8px 32px rgba(0, 0, 0, 0.18), + 0 2px 8px rgba(0, 0, 0, 0.1); + display: flex; + align-items: center; + border: 1.5px solid var(--accent-light); + opacity: 0.97; + will-change: transform; + font-family: + system-ui, + -apple-system, + sans-serif; +} + +column-chooser .scroll-pulse { + position: absolute; + left: 50%; + transform: translateX(-50%); + width: 32px; + height: 32px; + border-radius: 50%; + background: var(--accent); + opacity: 0; + pointer-events: none; + z-index: 10; + transition: opacity 0.15s; +} + +column-chooser .scroll-pulse.top { + top: 8px; +} + +column-chooser .scroll-pulse.bot { + bottom: 8px; +} + +column-chooser .scroll-pulse.active { + opacity: 0.18; + animation: column-chooser-pulse 0.8s ease-in-out infinite; +} + +@keyframes column-chooser-pulse { + 0%, + 100% { + transform: translateX(-50%) scale(1); + opacity: 0.18; + } + 50% { + transform: translateX(-50%) scale(1.5); + opacity: 0.07; + } +} + +column-chooser .modal-btn-primary { + color: white; +} + +column-chooser .modal-btn-primary:hover { + background: #1448c0; +} + +column-chooser .list-wrap::-webkit-scrollbar { + width: 5px; +} + +column-chooser .list-wrap::-webkit-scrollbar-track { + background: transparent; +} + +column-chooser .list-wrap::-webkit-scrollbar-thumb { + background: var(--rule); + border-radius: 99px; +} + +column-chooser input, +column-chooser textarea { + -webkit-user-select: auto; + user-select: auto; +} + +dialog.mobile-column-actions-dialog { + width: min(420px, calc(100vw - 32px)); + max-height: min(640px, calc(100vh - 32px)); +} + +.mobile-column-actions-dialog .modal-header { + padding: 20px 24px 16px; + justify-content: space-between; } .mobile-column-actions-dialog .list-wrap { flex: 1 1 auto; - min-height: 0; - overflow-y: auto; + padding: 0; overflow-x: hidden; position: relative; overscroll-behavior: contain; @@ -1142,102 +1794,12 @@ dialog.mobile-column-actions-dialog::backdrop { font-size: 0.85em; } -.mobile-column-actions-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.mobile-column-actions-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.mobile-column-actions-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.mobile-column-actions-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.mobile-column-actions-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - dialog.set-column-type-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(720px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.set-column-type-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.set-column-type-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .set-column-type-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.set-column-type-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.set-column-type-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; } .set-column-type-status, @@ -1259,8 +1821,6 @@ dialog.set-column-type-dialog::backdrop { } .set-column-type-options { - padding: 16px 24px 24px; - overflow-y: auto; display: grid; gap: 12px; } @@ -1302,60 +1862,6 @@ dialog.set-column-type-dialog::backdrop { font-size: 0.9rem; } -.set-column-type-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.set-column-type-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.set-column-type-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.set-column-type-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.set-column-type-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.set-column-type-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.set-column-type-dialog .btn-primary:hover { - background: #1949b8; -} - -.set-column-type-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - .row-mutation-status { margin: 0 0 0.75rem; padding: 8px 10px; @@ -1389,46 +1895,11 @@ button.table-insert-row svg { } dialog.row-delete-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(440px, calc(100vw - 32px)); - max-width: 95vw; - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.row-delete-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.row-delete-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .row-delete-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; justify-content: flex-start; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .row-delete-dialog .modal-title { @@ -1437,9 +1908,6 @@ dialog.row-delete-dialog::backdrop { gap: 0.35rem; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .row-delete-message, @@ -1471,94 +1939,12 @@ dialog.row-delete-dialog::backdrop { .row-delete-dialog .modal-footer { padding: 18px 20px 14px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); margin-top: 18px; } -.row-delete-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.row-delete-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.row-delete-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.row-delete-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.row-delete-dialog .btn-primary:hover { - background: #1949b8; -} - -.row-delete-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - dialog.row-edit-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(720px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.row-edit-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.row-edit-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.row-edit-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .row-edit-dialog .modal-title { @@ -1567,9 +1953,6 @@ dialog.row-edit-dialog::backdrop { gap: 0.35rem; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .row-edit-dialog .modal-title .row-dialog-action, @@ -1637,8 +2020,6 @@ dialog.row-edit-dialog::backdrop { .row-edit-fields { display: grid; gap: 14px; - padding: 16px 24px 24px; - overflow-y: auto; } .row-edit-fields[hidden], @@ -1918,8 +2299,6 @@ textarea.row-edit-input { .row-edit-bulk { display: grid; gap: 8px; - padding: 16px 24px 24px; - overflow-y: auto; } .row-edit-bulk-editor { @@ -1939,7 +2318,7 @@ textarea.row-edit-input { justify-content: flex-start; } -.row-edit-bulk-actions .btn { +.row-edit-bulk-actions .modal-btn { padding-left: 12px; padding-right: 12px; } @@ -2183,17 +2562,6 @@ datasette-autocomplete input[type="text"], max-width: 46rem; } -.row-edit-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - .row-edit-mode-link { color: var(--accent); font-size: 0.9rem; @@ -2204,84 +2572,14 @@ datasette-autocomplete input[type="text"], display: none; } -.row-edit-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.row-edit-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.row-edit-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.row-edit-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.row-edit-dialog .btn-primary:hover { - background: #1949b8; -} - -.row-edit-dialog .btn:disabled { +.row-edit-dialog .modal-btn:disabled { opacity: 0.55; cursor: not-allowed; } dialog.table-create-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(980px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.table-create-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.table-create-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.table-create-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .table-create-dialog .modal-title { @@ -2289,9 +2587,6 @@ dialog.table-create-dialog::backdrop { align-items: center; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .table-create-form { @@ -2319,8 +2614,6 @@ dialog.table-create-dialog::backdrop { .table-create-fields { display: grid; gap: 18px; - padding: 16px 24px 24px; - overflow-y: auto; } .table-create-field { @@ -2730,17 +3023,6 @@ select.table-create-input { outline-offset: 1px; } -.table-create-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - .table-create-mode-link { color: var(--accent); font-size: 0.9rem; @@ -2751,39 +3033,7 @@ select.table-create-input { display: none; } -.table-create-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.table-create-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.table-create-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.table-create-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.table-create-dialog .btn-primary:hover { - background: #1949b8; -} - -.table-create-dialog .btn:disabled, +.table-create-dialog .modal-btn:disabled, .table-create-add-column:disabled, .table-create-icon-button:disabled { opacity: 0.55; @@ -2791,46 +3041,8 @@ select.table-create-input { } dialog.table-alter-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; width: min(980px, calc(100vw - 32px)); - max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.table-alter-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.table-alter-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.table-alter-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; } .table-alter-dialog .modal-title { @@ -2838,9 +3050,6 @@ dialog.table-alter-dialog::backdrop { align-items: center; min-width: 0; max-width: 100%; - font-size: 1rem; - font-weight: 600; - color: var(--ink); } .table-alter-form { @@ -2868,8 +3077,6 @@ dialog.table-alter-dialog::backdrop { .table-alter-fields { display: grid; gap: 18px; - padding: 16px 24px 24px; - overflow-y: auto; } .table-alter-table-options { @@ -2903,8 +3110,6 @@ dialog.table-alter-dialog::backdrop { .table-alter-review { display: grid; gap: 12px; - overflow-y: auto; - padding: 16px 24px 24px; } .table-alter-review[hidden] { @@ -3198,72 +3403,29 @@ select.table-alter-input { outline-offset: 1px; } -.table-alter-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.table-alter-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.table-alter-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.table-alter-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.table-alter-dialog .btn-danger { +.table-alter-dialog .modal-btn-danger { background: #b91c1c; color: #fff; margin-right: auto; } -.table-alter-dialog .btn-danger:hover { +.table-alter-dialog .modal-btn-danger:hover { background: #991b1b; } -.table-alter-dialog .btn-danger:disabled, -.table-alter-dialog .btn-danger:disabled:hover { +.table-alter-dialog .modal-btn-danger:disabled, +.table-alter-dialog .modal-btn-danger:disabled:hover { background: #d98c8c; color: #fff; } -.table-alter-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.table-alter-dialog .btn-primary:hover { - background: #1949b8; -} - -.table-alter-dialog .btn-primary:disabled, -.table-alter-dialog .btn-primary:disabled:hover { +.table-alter-dialog .modal-btn-primary:disabled, +.table-alter-dialog .modal-btn-primary:disabled:hover { background: #a0aec0; color: #fff; } -.table-alter-dialog .btn:disabled, +.table-alter-dialog .modal-btn:disabled, .table-alter-add-column:disabled, .table-alter-icon-button:disabled { opacity: 0.55; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index 198641f3..29729f27 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -1,7 +1,9 @@ +let columnChooserInstanceCounter = 0; + class ColumnChooser extends HTMLElement { constructor() { super(); - this.attachShadow({ mode: "open" }); + this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`; // State this._items = []; @@ -26,375 +28,60 @@ class ColumnChooser extends HTMLElement { // Bound handlers this._onMove = this._onMove.bind(this); this._onUp = this._onUp.bind(this); + } - this.shadowRoot.innerHTML = ` - - - + connectedCallback() { + if (this._modal) return; + this.innerHTML = ` +
- Choose columns - + Choose columns +
- - + +
-
-
-
-
    +
    +
    +
    +
      - - - + + +
      -
      +       `; // DOM refs - this._dialog = this.shadowRoot.querySelector("dialog"); - this._listWrap = this.shadowRoot.getElementById("listWrap"); - this._dragList = this.shadowRoot.getElementById("dragList"); - this._pulseTop = this.shadowRoot.getElementById("pulseTop"); - this._pulseBot = this.shadowRoot.getElementById("pulseBot"); - this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); - this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); - this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); - this._applyBtn = this.shadowRoot.getElementById("applyBtn"); - this._countEl = this.shadowRoot.getElementById("selectedCount"); - this._footerEl = this.shadowRoot.getElementById("footerInfo"); + this._modal = this.querySelector("datasette-modal"); + this._listWrap = this.querySelector(".list-wrap"); + this._dragList = this.querySelector(".drag-list"); + this._pulseTop = this.querySelector(".scroll-pulse.top"); + this._pulseBot = this.querySelector(".scroll-pulse.bot"); + this._selectAllBtn = this.querySelector(".select-all"); + this._deselectAllBtn = this.querySelector(".deselect-all"); + this._cancelBtn = this.querySelector(".modal-btn-ghost"); + this._applyBtn = this.querySelector(".modal-btn-primary"); + this._countEl = this.querySelector(".modal-meta"); + this._footerEl = this.querySelector(".footer-info"); // Event listeners this._selectAllBtn.addEventListener("click", () => this._selectAll()); this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); - this._cancelBtn.addEventListener("click", () => this._close()); + this._cancelBtn.addEventListener("click", () => + this._modal.requestClose("cancel"), + ); this._applyBtn.addEventListener("click", () => this._apply()); - this._dialog.addEventListener("click", (e) => { - if (e.target === this._dialog) this._close(); - }); - this._dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this._close(); - }); + this._modal.beforeClose = () => { + this._items = this._savedItems ? [...this._savedItems] : this._items; + this._checked = this._savedChecked + ? new Set(this._savedChecked) + : this._checked; + return true; + }; } /** @@ -414,19 +101,11 @@ class ColumnChooser extends HTMLElement { this._savedChecked = new Set(this._checked); this._render(); - this._dialog.showModal(); + this._modal.show(); } // ── Internal methods ── - _close() { - this._items = this._savedItems ? [...this._savedItems] : this._items; - this._checked = this._savedChecked - ? new Set(this._savedChecked) - : this._checked; - this._dialog.close(); - } - _selectAll() { this._items.forEach((col) => this._checked.add(col)); this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { @@ -445,7 +124,7 @@ class ColumnChooser extends HTMLElement { _apply() { const selected = this._items.filter((col) => this._checked.has(col)); - this._dialog.close(); + this._modal.close(); if (this._onApply) { this._onApply(selected); } @@ -472,11 +151,13 @@ class ColumnChooser extends HTMLElement { - ${col} +
      `; + li.querySelector(".drag-item-label").textContent = col; + li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); @@ -509,7 +190,7 @@ class ColumnChooser extends HTMLElement { this._ghostOffX = e.clientX - rect.left; this._ghostOffY = e.clientY - rect.top; - // Build ghost inside shadow DOM + // Keep the drag preview inside the dialog so it stays above the backdrop. this._ghost = document.createElement("div"); this._ghost.className = "drag-ghost"; this._ghost.style.width = rect.width + "px"; @@ -518,7 +199,7 @@ class ColumnChooser extends HTMLElement { this._ghost.querySelector(".drop-indicator")?.remove(); const h = this._ghost.querySelector(".drag-handle"); if (h) h.style.color = "var(--accent)"; - this.shadowRoot.appendChild(this._ghost); + this._modal.dialog.appendChild(this._ghost); srcEl.classList.add("is-dragging"); this._positionGhost(e.clientX, e.clientY); diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 9e8b93f6..0f61ebd6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -915,6 +915,7 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; + state.modal.busy = isSaving; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { @@ -2043,8 +2044,7 @@ async function createTableFromDataPreview(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (tableUrl) { location.href = tableUrl; } else { @@ -2118,8 +2118,7 @@ async function saveTableCreateDialog(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (tableUrl) { location.href = tableUrl; } else { @@ -2141,18 +2140,6 @@ function confirmDiscardTableCreateChanges(state) { return window.confirm("Discard this new table?"); } -function closeTableCreateDialogIfConfirmed(state) { - if (!state || state.isSaving) { - return false; - } - if (!confirmDiscardTableCreateChanges(state)) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - function ensureTableCreateDialog(manager) { if (tableCreateDialogState) { return tableCreateDialogState; @@ -2161,7 +2148,8 @@ function ensureTableCreateDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = TABLE_CREATE_DIALOG_ID; dialog.className = "table-create-dialog"; dialog.setAttribute("aria-labelledby", "table-create-title"); @@ -2171,7 +2159,7 @@ function ensureTableCreateDialog(manager) {
      -
      +
      @@ -2198,14 +2186,15 @@ function ensureTableCreateDialog(manager) {
      Create table from data - - + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); tableCreateDialogState = { + modal: modal, dialog: dialog, form: dialog.querySelector(".table-create-form"), title: dialog.querySelector(".modal-title"), @@ -2225,8 +2214,6 @@ function ensureTableCreateDialog(manager) { manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), - currentButton: null, - shouldRestoreFocus: true, isSaving: false, mode: "manual", dataPreviewRows: null, @@ -2266,7 +2253,7 @@ function ensureTableCreateDialog(manager) { tableCreateDialogState.dataTextarea.focus(); return; } - closeTableCreateDialogIfConfirmed(tableCreateDialogState); + modal.requestClose("cancel"); }); tableCreateDialogState.createFromDataLink.addEventListener( @@ -2364,36 +2351,14 @@ function ensureTableCreateDialog(manager) { updateTableCreateDialogButtons(tableCreateDialogState); }); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeTableCreateDialogIfConfirmed(tableCreateDialogState); - } - }); - - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { - return; - } - ev.preventDefault(); - closeTableCreateDialogIfConfirmed(tableCreateDialogState); - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeTableCreateDialogIfConfirmed(tableCreateDialogState); - }); + modal.beforeClose = function (source) { + return confirmDiscardTableCreateChanges(tableCreateDialogState); + }; dialog.addEventListener("close", function () { var state = tableCreateDialogState; clearTableCreateDialogError(state); setTableCreateDialogSaving(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } }); return tableCreateDialogState; @@ -2414,15 +2379,12 @@ function openTableCreateDialog(button, manager) { menu.open = false; } state.manager = manager; - state.currentButton = button; - state.shouldRestoreFocus = true; + state.title.textContent = "Create a table in " + data.databaseName; clearTableCreateDialogError(state); resetTableCreateDialog(state); loadTableCreateForeignKeyTargets(state); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); state.tableName.focus(); } @@ -2448,6 +2410,7 @@ function initTableCreateActions(manager) { function setRowDeleteDialogBusy(state, isBusy) { state.isBusy = isBusy; + state.modal.busy = isBusy; state.confirmButton.disabled = isBusy; state.cancelButton.disabled = isBusy; state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row"; @@ -2694,6 +2657,7 @@ function showTableAlterDialogError(state, message) { function setTableAlterDialogSaving(state, isSaving) { state.isSaving = isSaving; + state.modal.busy = isSaving; state.cancelButton.disabled = isSaving; state.addColumnButton.disabled = isSaving; state.backButton.disabled = isSaving; @@ -3829,8 +3793,7 @@ async function applyTableAlterChanges(state, result) { result.columnTypeAssignments || [], tableUrl, ); - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (tableAlterResultRenamesTable(result) && tableUrl) { window.location.href = tableUrl; } else { @@ -3891,8 +3854,7 @@ async function dropTableFromAlterDialog(state) { if (!response.ok || (responseData && responseData.ok === false)) { throw rowMutationRequestError(response, responseData); } - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); window.location.href = tableAlterDatabaseUrl() || "/"; } catch (error) { setTableAlterDialogSaving(state, false); @@ -3928,27 +3890,6 @@ function confirmDiscardTableAlterChanges(state) { return window.confirm("Discard table changes?"); } -function closeTableAlterDialogIfConfirmed(state) { - if (!state || state.isSaving) { - return false; - } - if (!confirmDiscardTableAlterChanges(state)) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - -function closeTableAlterDialog(state) { - if (!state || state.isSaving) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - function ensureTableAlterDialog(manager) { if (tableAlterDialogState) { return tableAlterDialogState; @@ -3957,7 +3898,8 @@ function ensureTableAlterDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = TABLE_ALTER_DIALOG_ID; dialog.className = "table-alter-dialog"; dialog.setAttribute("aria-labelledby", "table-alter-title"); @@ -3967,7 +3909,7 @@ function ensureTableAlterDialog(manager) {
      -
      +
      - +
      - - - - + + + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); tableAlterDialogState = { + modal: modal, dialog: dialog, form: dialog.querySelector(".table-alter-form"), title: dialog.querySelector(".modal-title"), @@ -4012,8 +3955,6 @@ function ensureTableAlterDialog(manager) { dropButton: dialog.querySelector(".table-alter-drop"), cancelButton: dialog.querySelector(".table-alter-cancel"), saveButton: dialog.querySelector(".table-alter-save"), - currentButton: null, - shouldRestoreFocus: true, isSaving: false, initialSignature: "", originalTableName: "", @@ -4055,7 +3996,7 @@ function ensureTableAlterDialog(manager) { }); tableAlterDialogState.cancelButton.addEventListener("click", function () { - closeTableAlterDialog(tableAlterDialogState); + modal.requestClose("cancel"); }); tableAlterDialogState.dropButton.addEventListener("click", function () { @@ -4076,36 +4017,17 @@ function ensureTableAlterDialog(manager) { } }); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeTableAlterDialogIfConfirmed(tableAlterDialogState); - } - }); - - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { - return; - } - ev.preventDefault(); - closeTableAlterDialogIfConfirmed(tableAlterDialogState); - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeTableAlterDialogIfConfirmed(tableAlterDialogState); - }); + modal.beforeClose = function (source) { + return ( + source === "cancel" || + confirmDiscardTableAlterChanges(tableAlterDialogState) + ); + }; dialog.addEventListener("close", function () { var state = tableAlterDialogState; clearTableAlterDialogError(state); setTableAlterDialogSaving(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } }); return tableAlterDialogState; @@ -4126,8 +4048,7 @@ function openTableAlterDialog(button, manager) { menu.open = false; } state.manager = manager; - state.currentButton = button; - state.shouldRestoreFocus = true; + state.title.textContent = "Alter table " + data.tableName; clearTableAlterDialogError(state); resetTableAlterDialog(state, data); @@ -4137,9 +4058,7 @@ function openTableAlterDialog(button, manager) { tableAlterForeignKeyTargetsUrl(), { filterByType: false }, ); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); var firstName = state.columnList.querySelector(".table-alter-column-name"); if (firstName) { firstName.focus(); @@ -4442,7 +4361,8 @@ function ensureRowDeleteDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = ROW_DELETE_DIALOG_ID; dialog.className = "row-delete-dialog"; dialog.setAttribute("aria-labelledby", "row-delete-title"); @@ -4454,13 +4374,14 @@ function ensureRowDeleteDialog(manager) {

      Delete row ?

      - - + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); rowDeleteDialogState = { + modal: modal, dialog: dialog, title: dialog.querySelector(".modal-title"), message: dialog.querySelector(".row-delete-message"), @@ -4473,21 +4394,10 @@ function ensureRowDeleteDialog(manager) { currentPkPath: null, manager: manager, isBusy: false, - shouldRestoreFocus: true, }; rowDeleteDialogState.cancelButton.addEventListener("click", function () { - if (!rowDeleteDialogState.isBusy) { - rowDeleteDialogState.shouldRestoreFocus = true; - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !rowDeleteDialogState.isBusy) { - rowDeleteDialogState.shouldRestoreFocus = true; - dialog.close(); - } + modal.requestClose("cancel"); }); dialog.addEventListener("keydown", function (ev) { @@ -4499,25 +4409,6 @@ function ensureRowDeleteDialog(manager) { if (!rowDeleteDialogState.isBusy) { rowDeleteDialogState.confirmButton.click(); } - return; - } - if (ev.key !== "Escape") { - return; - } - if (rowDeleteDialogState.isBusy) { - ev.preventDefault(); - return; - } - ev.preventDefault(); - rowDeleteDialogState.shouldRestoreFocus = true; - dialog.close(); - }); - - dialog.addEventListener("cancel", function (ev) { - if (rowDeleteDialogState.isBusy) { - ev.preventDefault(); - } else { - rowDeleteDialogState.shouldRestoreFocus = true; } }); @@ -4525,13 +4416,6 @@ function ensureRowDeleteDialog(manager) { var state = rowDeleteDialogState; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } }); rowDeleteDialogState.confirmButton.addEventListener( @@ -4558,8 +4442,7 @@ function ensureRowDeleteDialog(manager) { throw rowMutationRequestError(response, data); } if (data && data.redirect) { - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); location.href = data.redirect; return; } @@ -4571,8 +4454,7 @@ function ensureRowDeleteDialog(manager) { var statusMessage = state.currentPkPath ? "Deleted row " + state.currentPkPath + "." : "Deleted row."; - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); state.currentRow.remove(); showRowMutationStatus(state.manager, statusMessage, false); if (focusTarget && document.contains(focusTarget)) { @@ -4601,11 +4483,9 @@ function openRowDeleteDialog(button, manager) { } state.manager = manager; - state.currentButton = button; state.currentRow = row; state.currentDeleteUrl = rowDeleteUrl(row); state.currentPkPath = rowDisplayLabel(row); - state.shouldRestoreFocus = true; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); @@ -4617,9 +4497,7 @@ function openRowDeleteDialog(button, manager) { ); state.rowId.textContent = state.currentPkPath || "this row"; - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); state.confirmButton.focus(); } @@ -5694,6 +5572,7 @@ function setRowEditDialogLoading(state, isLoading) { function setRowEditDialogSaving(state, isSaving) { state.isSaving = isSaving; + state.modal.busy = isSaving; updateRowEditDialogButtons(state); } @@ -5911,18 +5790,6 @@ function confirmDiscardRowEditChanges(state) { return window.confirm(message); } -function closeRowEditDialogIfConfirmed(state) { - if (!state || state.isSaving) { - return false; - } - if (!confirmDiscardRowEditChanges(state)) { - return false; - } - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; -} - function setRowInsertDialogTitle(state) { var insertData = tableInsertData() || {}; var title = rowEditIsMultipleInsert(state) @@ -6748,38 +6615,6 @@ async function insertBulkPreviewRows(state) { } } -function scheduleCloseRowEditDialogIfConfirmed(state) { - // Fix for an issue in Safari where hitting Esc would show - // the confirm() prompt asking if state should be discarded - // but the Esc key press would then cancel that dialog too. - // Wait for keyup, then move the confirm() to a fresh timer tick. - if (!state || state.isSaving || state.isClosePending) { - return false; - } - if (!rowEditDialogHasChanges(state)) { - state.shouldRestoreFocus = true; - state.dialog.close(); - return true; - } - state.isClosePending = true; - var closeAfterKeyup = function () { - if (!state.isClosePending) { - return; - } - state.isClosePending = false; - closeRowEditDialogIfConfirmed(state); - }; - var onKeyup = function (ev) { - if (ev.key !== "Escape") { - return; - } - document.removeEventListener("keyup", onKeyup, true); - setTimeout(closeAfterKeyup, 0); - }; - document.addEventListener("keyup", onKeyup, true); - return true; -} - function findDataRowElement(root, rowId) { var elements = root.querySelectorAll("[data-row]"); for (var i = 0; i < elements.length; i += 1) { @@ -6869,9 +6704,8 @@ async function saveRowEditDialog(state) { } var formValues = collectRowFormValues(state); if (state.mode === "edit" && !Object.keys(formValues).length) { - state.shouldRestoreFocus = true; hideRowMutationStatus(); - state.dialog.close(); + state.modal.close(); return; } var payload = @@ -6904,9 +6738,8 @@ async function saveRowEditDialog(state) { insertedRowData, insertData.primaryKeys || [], ); - state.shouldRestoreFocus = false; if (!insertedRowId) { - state.dialog.close(); + state.modal.close({ restoreFocus: false }); var missingIdStatus = showRowMutationStatus( state.manager, "Inserted row. Refresh the page to see it.", @@ -6922,7 +6755,7 @@ async function saveRowEditDialog(state) { try { insertedRow = await fetchUpdatedRowElement(state); } catch (_error) { - state.dialog.close(); + state.modal.close({ restoreFocus: false }); var refreshFailedStatus = showRowMutationStatus( state.manager, "Inserted row, but could not refresh the table row. Refresh the page to see it.", @@ -6937,7 +6770,7 @@ async function saveRowEditDialog(state) { rowTitleLabel(insertedRow), ); var addedRow = addInsertedRowToPage(insertedRow); - state.dialog.close(); + state.modal.close({ restoreFocus: false }); showRowMutationStatus(state.manager, insertedStatusMessage, false); if (addedRow) { var insertedFocusTarget = @@ -6946,7 +6779,7 @@ async function saveRowEditDialog(state) { insertedFocusTarget.focus(); } } else { - state.dialog.close(); + state.modal.close({ restoreFocus: false }); var filteredStatus = showRowMutationStatus( state.manager, "Inserted row. It does not match the current filters.", @@ -6958,8 +6791,7 @@ async function saveRowEditDialog(state) { } if (isRowPage()) { - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); location.reload(); return; } @@ -6995,8 +6827,7 @@ async function saveRowEditDialog(state) { ); } - state.shouldRestoreFocus = false; - state.dialog.close(); + state.modal.close({ restoreFocus: false }); if (focusTarget && document.contains(focusTarget)) { focusTarget.focus(); } @@ -7140,7 +6971,8 @@ function ensureRowEditDialog(manager) { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = ROW_EDIT_DIALOG_ID; dialog.className = "row-edit-dialog"; dialog.setAttribute("aria-labelledby", "row-edit-title"); @@ -7152,8 +6984,8 @@ function ensureRowEditDialog(manager) {

      Loading row...

      -
      - @@ -7183,14 +7015,15 @@ function ensureRowEditDialog(manager) {
      - - + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); rowEditDialogState = { + modal: modal, dialog: dialog, form: dialog.querySelector(".row-edit-form"), title: dialog.querySelector(".modal-title"), @@ -7221,7 +7054,6 @@ function ensureRowEditDialog(manager) { singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), - currentButton: null, currentRow: null, currentRowId: null, currentPkPath: null, @@ -7249,9 +7081,7 @@ function ensureRowEditDialog(manager) { manager: manager, isLoading: false, isSaving: false, - isClosePending: false, hasLoaded: false, - shouldRestoreFocus: true, }; rowEditDialogState.form.addEventListener("submit", function (ev) { @@ -7271,10 +7101,7 @@ function ensureRowEditDialog(manager) { rowEditDialogState.bulkInsertTextarea.focus(); return; } - if (!rowEditDialogState.isSaving) { - rowEditDialogState.shouldRestoreFocus = true; - dialog.close(); - } + modal.requestClose("cancel"); }); rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { @@ -7393,31 +7220,17 @@ function ensureRowEditDialog(manager) { }, ); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeRowEditDialogIfConfirmed(rowEditDialogState); - } - }); - - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { - return; - } - ev.preventDefault(); - scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); - }); + modal.beforeClose = function (source) { + return ( + source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState) + ); + }; dialog.addEventListener("close", function () { var state = rowEditDialogState; var shouldReloadOnClose = state.shouldReloadOnClose; var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; - state.isClosePending = false; state.bulkInsertLiveValidationError = null; state.shouldReloadOnClose = false; state.redirectOnCloseUrl = null; @@ -7430,13 +7243,6 @@ function ensureRowEditDialog(manager) { destroyRowEditFields(state); setRowEditDialogLoading(state, false); setRowEditDialogSaving(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } if (shouldReloadOnClose) { if (redirectOnCloseUrl) { location.href = redirectOnCloseUrl; @@ -7461,7 +7267,6 @@ async function openRowEditDialog(button, manager) { state.manager = manager; state.mode = "edit"; - state.currentButton = button; state.currentRow = row; state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); @@ -7478,7 +7283,7 @@ async function openRowEditDialog(button, manager) { } else { state.form.removeAttribute("action"); } - state.shouldRestoreFocus = true; + state.hasLoaded = false; state.loadId += 1; var loadId = state.loadId; @@ -7497,9 +7302,7 @@ async function openRowEditDialog(button, manager) { state.summary.textContent = ""; syncRowEditInsertModeUi(state); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); state.cancelButton.focus(); try { @@ -7539,7 +7342,6 @@ function openRowInsertDialog(button, manager) { state.manager = manager; state.mode = "insert"; - state.currentButton = button; state.currentRow = null; state.currentRowId = null; state.currentPkPath = null; @@ -7554,7 +7356,7 @@ function openRowInsertDialog(button, manager) { state.shouldReloadOnClose = false; state.redirectOnCloseUrl = null; resetBulkInsertPreview(state); - state.shouldRestoreFocus = true; + state.hasLoaded = false; state.loadId += 1; @@ -7576,9 +7378,7 @@ function openRowInsertDialog(button, manager) { state.summary.textContent = ""; syncRowEditInsertModeUi(state); - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show({ returnFocusTo: button }); renderRowInsertFields(state, insertData); } diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js deleted file mode 100644 index 0e6e2c29..00000000 --- a/datasette/static/json-format-highlight-1.0.1.js +++ /dev/null @@ -1,56 +0,0 @@ -/* -https://github.com/luyilin/json-format-highlight -From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js -MIT Licensed -*/ -(function (global, factory) { - typeof exports === "object" && typeof module !== "undefined" - ? (module.exports = factory()) - : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); -})(this, function () { - "use strict"; - - var defaultColors = { - keyColor: "dimgray", - numberColor: "lightskyblue", - stringColor: "lightcoral", - trueColor: "lightseagreen", - falseColor: "#f66578", - nullColor: "cornflowerblue", - }; - - function index(json, colorOptions) { - if (colorOptions === void 0) colorOptions = {}; - - if (!json) { - return; - } - if (typeof json !== "string") { - json = JSON.stringify(json, null, 2); - } - var colors = Object.assign({}, defaultColors, colorOptions); - json = json.replace(/&/g, "&").replace(//g, ">"); - return json.replace( - /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, - function (match) { - var color = colors.numberColor; - if (/^"/.test(match)) { - color = /:$/.test(match) ? colors.keyColor : colors.stringColor; - } else { - color = /true/.test(match) - ? colors.trueColor - : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; - } - return '' + match + ""; - }, - ); - } - - return index; -}); diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js index a386b1fc..e94ee865 100644 --- a/datasette/static/mobile-column-actions.js +++ b/datasette/static/mobile-column-actions.js @@ -66,7 +66,8 @@ function initMobileColumnActions(manager) { return; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.className = "mobile-column-actions-dialog"; dialog.id = MOBILE_COLUMN_DIALOG_ID; dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); @@ -75,13 +76,13 @@ function initMobileColumnActions(manager) { Column actions
      -
      +
      Tap a column to reveal actions. - +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); triggerButton.setAttribute("aria-haspopup", "dialog"); triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); @@ -91,7 +92,6 @@ function initMobileColumnActions(manager) { var listWrap = dialog.querySelector(".mobile-column-list"); var doneButton = dialog.querySelector(".mobile-column-actions-done"); var expandedSectionId = null; - var shouldRestoreFocus = true; function updateExpandedSection() { Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { @@ -128,16 +128,7 @@ function initMobileColumnActions(manager) { } function closeDialog(options) { - options = options || {}; - shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } - } + modal.close(options); } function renderDialog() { @@ -166,7 +157,8 @@ function initMobileColumnActions(manager) { topActions.className = "mobile-column-top-actions"; var showAllColumns = document.createElement("a"); - showAllColumns.className = "btn btn-ghost mobile-column-top-action"; + showAllColumns.className = + "modal-btn modal-btn-ghost mobile-column-top-action"; showAllColumns.href = manager.columnActions.showAllColumnsUrl(); showAllColumns.textContent = "Show all columns"; @@ -265,9 +257,7 @@ function initMobileColumnActions(manager) { if (!renderDialog()) { return; } - if (!dialog.open) { - dialog.showModal(); - } + modal.show({ returnFocusTo: triggerButton }); triggerButton.setAttribute("aria-expanded", "true"); var focusTarget = dialog.querySelector(".mobile-column-top-action") || @@ -288,22 +278,8 @@ function initMobileColumnActions(manager) { closeDialog(); }); - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog) { - closeDialog(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - ev.preventDefault(); - closeDialog(); - }); - dialog.addEventListener("close", function () { triggerButton.setAttribute("aria-expanded", "false"); - if (shouldRestoreFocus) { - triggerButton.focus(); - } }); window.addEventListener("resize", function () { diff --git a/datasette/static/modal.js b/datasette/static/modal.js new file mode 100644 index 00000000..ca9ab353 --- /dev/null +++ b/datasette/static/modal.js @@ -0,0 +1,167 @@ +// Shared lifecycle for native modal dialogs. +(() => { + class DatasetteModal extends HTMLElement { + constructor() { + super(); + this.beforeClose = null; + this._busy = false; + this._restoreFocus = true; + this._returnFocusTo = null; + this._escapeCleanup = null; + this._escapeTimer = null; + } + + static create() { + const modal = document.createElement("datasette-modal"); + modal.appendChild(document.createElement("dialog")); + return modal; + } + + get dialog() { + return this.querySelector(":scope > dialog"); + } + + get busy() { + return this._busy; + } + + set busy(value) { + this._busy = !!value; + if (this.dialog) { + this.dialog.setAttribute("aria-busy", String(this._busy)); + } + } + + connectedCallback() { + const dialog = this.dialog; + if (!dialog) return; + dialog.classList.add("datasette-modal"); + this._listeners?.abort(); + this._listeners = new AbortController(); + const options = { signal: this._listeners.signal }; + let backdropPointerDown = false; + const outside = (event) => { + const rect = dialog.getBoundingClientRect(); + return ( + event.target === dialog && + (event.clientX < rect.left || + event.clientX > rect.right || + event.clientY < rect.top || + event.clientY > rect.bottom) + ); + }; + dialog.addEventListener( + "pointerdown", + (event) => { + backdropPointerDown = outside(event); + }, + options, + ); + dialog.addEventListener( + "click", + (event) => { + if (backdropPointerDown && outside(event)) + this.requestClose("backdrop"); + backdropPointerDown = false; + }, + options, + ); + dialog.addEventListener( + "keydown", + (event) => { + if (event.key !== "Escape" || event.defaultPrevented) return; + // A nested native dialog or plugin picker gets first refusal. + if (event.target.closest("dialog") !== dialog) return; + event.preventDefault(); + if (this.busy || this._escapeCleanup || this._escapeTimer !== null) + return; + // Safari can otherwise use this Escape press to cancel confirm() too. + // Only keyboard dismissals wait for keyup; native cancel events needn't. + const onKeyup = (up) => { + if (up.key !== "Escape") return; + this._escapeCleanup(); + this._escapeCleanup = null; + this._escapeTimer = setTimeout(() => { + this._escapeTimer = null; + this.requestClose("escape"); + }, 0); + }; + this.ownerDocument.addEventListener("keyup", onKeyup, true); + this._escapeCleanup = () => + this.ownerDocument.removeEventListener("keyup", onKeyup, true); + }, + options, + ); + dialog.addEventListener( + "cancel", + (event) => { + if (event.target !== dialog) return; + event.preventDefault(); + if (!this._escapeCleanup && this._escapeTimer === null) + this.requestClose("escape"); + }, + options, + ); + dialog.addEventListener( + "close", + (event) => { + if (event.target !== dialog || dialog.open) return; + this._clearPendingClose(); + this.busy = false; + if (this._restoreFocus && this._returnFocusTo?.isConnected) { + // Menu actions may have become hidden while the dialog was open. + const details = this._returnFocusTo.closest("details:not([open])"); + const target = + details?.querySelector("summary") || this._returnFocusTo; + target.focus({ preventScroll: true }); + } + this._returnFocusTo = null; + }, + options, + ); + } + + disconnectedCallback() { + this._listeners?.abort(); + this._clearPendingClose(); + this._returnFocusTo = null; + if (this.dialog?.open) this.dialog.close(); + this.busy = false; + } + + _clearPendingClose() { + this._escapeCleanup?.(); + this._escapeCleanup = null; + clearTimeout(this._escapeTimer); + this._escapeTimer = null; + } + + show({ returnFocusTo, initialFocus } = {}) { + const dialog = this.dialog; + if (!dialog.open) { + this._clearPendingClose(); + this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement; + this._restoreFocus = true; + dialog.showModal(); + } + if (typeof initialFocus === "function") initialFocus(); + else initialFocus?.focus(); + } + + requestClose(source = "cancel") { + if (!this.dialog.open || this.busy) return false; + if (this.beforeClose && this.beforeClose(source) === false) return false; + this.close(); + return true; + } + + close({ restoreFocus = true } = {}) { + this._clearPendingClose(); + this._restoreFocus = restoreFocus; + this.dialog.close(); + } + } + + customElements.define("datasette-modal", DatasetteModal); + window.DatasetteModal = DatasetteModal; +})(); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index ec2d23d8..df2516f9 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -10,277 +10,22 @@ class NavigationSearch extends HTMLElement { this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; this.statusId = `navigation-search-status-${this.instanceId}`; this.titleId = `navigation-search-title-${this.instanceId}`; - this.attachShadow({ mode: "open" }); this.selectedIndex = -1; this.matches = []; this.renderedMatches = []; this.debounceTimer = null; - this.restoreFocusTarget = null; - this.shouldRestoreFocus = true; + } + connectedCallback() { + if (this._initialized) return; + this._initialized = true; this.render(); this.setupEventListeners(); } render() { - this.shadowRoot.innerHTML = ` - - - + this.innerHTML = ` +
      @@ -302,23 +47,22 @@ class NavigationSearch extends HTMLElement { >
      -
      +
      ↑ ↓ Navigate Enter Select Esc Close
      - + `; } setupEventListeners() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const closeButton = this.shadowRoot.querySelector(".close-search"); - const resultsContainer = - this.shadowRoot.querySelector(".results-container"); + const dialog = this.querySelector("dialog"); + const input = this.querySelector(".search-input"); + const closeButton = this.querySelector(".close-search"); + const resultsContainer = this.querySelector(".results-container"); // Global keyboard listener for "/" document.addEventListener("keydown", (e) => { @@ -355,8 +99,6 @@ class NavigationSearch extends HTMLElement { } else if (e.key === "Enter") { e.preventDefault(); this.selectCurrentItem(); - } else if (e.key === "Escape") { - this.closeMenu(); } }); @@ -380,18 +122,6 @@ class NavigationSearch extends HTMLElement { } }); - // Close on backdrop click - dialog.addEventListener("click", (e) => { - if (e.target === dialog) { - this.closeMenu(); - } - }); - - dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this.closeMenu(); - }); - dialog.addEventListener("close", () => { this.onMenuClosed(); }); @@ -432,19 +162,6 @@ class NavigationSearch extends HTMLElement { } } - focusRestoreTarget(trigger) { - if (trigger && typeof trigger.focus === "function") { - return trigger; - } - if ( - document.activeElement && - typeof document.activeElement.focus === "function" - ) { - return document.activeElement; - } - return null; - } - setNavigationTriggersExpanded(expanded) { if (typeof document.querySelectorAll !== "function") { return; @@ -465,8 +182,8 @@ class NavigationSearch extends HTMLElement { } updateComboboxState() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); + const dialog = this.querySelector("dialog"); + const input = this.querySelector(".search-input"); const matches = this.renderedMatches || []; this.setElementAttribute( input, @@ -491,7 +208,7 @@ class NavigationSearch extends HTMLElement { } setStatus(message) { - const status = this.shadowRoot.querySelector(`#${this.statusId}`); + const status = this.querySelector(`#${this.statusId}`); if (status) { status.textContent = message || ""; } @@ -701,7 +418,7 @@ class NavigationSearch extends HTMLElement { section.render(node, { navigationSearch: this, container, - input: this.shadowRoot.querySelector(".search-input"), + input: this.querySelector(".search-input"), }); }); } @@ -740,8 +457,8 @@ class NavigationSearch extends HTMLElement { } renderResults() { - const container = this.shadowRoot.querySelector(".results-container"); - const input = this.shadowRoot.querySelector(".search-input"); + const container = this.querySelector(".results-container"); + const input = this.querySelector(".search-input"); const showStartContent = !input.value.trim(); const jumpSections = showStartContent ? this.jumpSections() : []; const startBlock = showStartContent @@ -853,18 +570,15 @@ class NavigationSearch extends HTMLElement { } } - openMenu(trigger) { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); + openMenu(returnFocusTo) { + const input = this.querySelector(".search-input"); - this.restoreFocusTarget = this.focusRestoreTarget(trigger); - this.shouldRestoreFocus = true; - if (!dialog.open) { - dialog.showModal(); - } + this.querySelector("datasette-modal").show({ + returnFocusTo, + initialFocus: input, + }); this.setNavigationTriggersExpanded(true); input.value = ""; - input.focus(); // Reset state, then populate the default jump list. this.matches = []; @@ -874,29 +588,15 @@ class NavigationSearch extends HTMLElement { } closeMenu(options = {}) { - const dialog = this.shadowRoot.querySelector("dialog"); - this.shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - this.onMenuClosed(); - } + this.querySelector("datasette-modal").close(options); } onMenuClosed() { - const input = this.shadowRoot.querySelector(".search-input"); + const input = this.querySelector(".search-input"); this.setElementAttribute(input, "aria-expanded", "false"); this.removeElementAttribute(input, "aria-activedescendant"); this.setNavigationTriggersExpanded(false); this.setStatus(""); - if ( - this.shouldRestoreFocus && - this.restoreFocusTarget && - typeof this.restoreFocusTarget.focus === "function" - ) { - this.restoreFocusTarget.focus(); - } - this.restoreFocusTarget = null; } escapeHtml(text) { diff --git a/datasette/static/table.js b/datasette/static/table.js index 74a96d8e..6b3e619c 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -157,6 +157,7 @@ function createSetColumnTypeOption(value, name, description, checked) { function setSetColumnTypeDialogBusy(state, isBusy) { state.isBusy = isBusy; + state.modal.busy = isBusy; state.saveButton.disabled = isBusy; state.cancelButton.disabled = isBusy; Array.from( @@ -185,7 +186,8 @@ function ensureSetColumnTypeDialog() { return null; } - var dialog = document.createElement("dialog"); + var modal = DatasetteModal.create(); + var dialog = modal.dialog; dialog.id = SET_COLUMN_TYPE_DIALOG_ID; dialog.className = "set-column-type-dialog"; dialog.setAttribute("aria-labelledby", "set-column-type-title"); @@ -196,16 +198,17 @@ function ensureSetColumnTypeDialog() {

      -
      +
      - - + +
      `; - document.body.appendChild(dialog); + document.body.appendChild(modal); setColumnTypeDialogState = { + modal: modal, dialog: dialog, meta: dialog.querySelector(".modal-meta"), status: dialog.querySelector(".set-column-type-status"), @@ -220,21 +223,7 @@ function ensureSetColumnTypeDialog() { }; setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - if (!setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - if (setColumnTypeDialogState.isBusy) { - ev.preventDefault(); - } + modal.requestClose("cancel"); }); dialog.addEventListener("close", function () { @@ -242,49 +231,52 @@ function ensureSetColumnTypeDialog() { setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); }); - setColumnTypeDialogState.saveButton.addEventListener("click", async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; + setColumnTypeDialogState.saveButton.addEventListener( + "click", + async function () { + var state = setColumnTypeDialogState; + var selected = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + var selectedType = selected ? selected.value : ""; + var currentType = state.currentConfig.current + ? state.currentConfig.current.type + : ""; - if (selectedType === currentType) { - state.dialog.close(); - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); + if (selectedType === currentType) { + state.modal.close(); + return; } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }); + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, true); + + var payload = { + column: state.currentColumn, + column_type: selectedType ? { type: selectedType } : null, + }; + + try { + var response = await fetch(getSetColumnTypeData().path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + var message = (data.errors || ["Request failed"]).join(" "); + throw new Error(message); + } + location.reload(); + } catch (error) { + setSetColumnTypeDialogBusy(state, false); + showSetColumnTypeDialogError(state, error.message || "Request failed"); + } + }, + ); return setColumnTypeDialogState; } @@ -341,9 +333,7 @@ function openSetColumnTypeDialog(th) { state.optionsWrap.appendChild(emptyState); } - if (!state.dialog.open) { - state.dialog.showModal(); - } + state.modal.show(); var selectedOption = state.dialog.querySelector( 'input[name="set-column-type-choice"]:checked', ); @@ -367,9 +357,10 @@ function shouldShowShowAllColumns() { function hasMultipleVisibleColumns(manager) { return ( - Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( - (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", - ).length > 1 + Array.from( + document.querySelectorAll(manager.selectors.tableHeaders), + ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1") + .length > 1 ); } @@ -649,10 +640,12 @@ function filterRowNumberFromName(name) { } function nextFilterRowNumber(manager) { - return filterRowsWithControls(manager).reduce((max, row) => { - var column = row.querySelector("select"); - return Math.max(max, filterRowNumberFromName(column && column.name)); - }, 0) + 1; + return ( + filterRowsWithControls(manager).reduce((max, row) => { + var column = row.querySelector("select"); + return Math.max(max, filterRowNumberFromName(column && column.name)); + }, 0) + 1 + ); } function setFilterRowNumber(row, number) { @@ -679,9 +672,11 @@ function updateFilterRowButtons(manager) { if (addButton) { addButton.hidden = index !== rows.length - 1 || !column.value; } - var visibleButtonCount = [removeButton, addButton].filter(function (button) { - return button && !button.hidden; - }).length; + var visibleButtonCount = [removeButton, addButton].filter( + function (button) { + return button && !button.hidden; + }, + ).length; row.classList.toggle( "filter-controls-row-has-buttons", visibleButtonCount > 0, @@ -703,7 +698,9 @@ function cloneFilterRow(row) { clone.querySelector(".filter-op select").name = "_filter_op"; clone.querySelector("input.filter-value").name = "_filter_value"; resetFilterRow(clone); - clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove()); + clone + .querySelectorAll(".filter-row-icon") + .forEach((button) => button.remove()); return clone; } @@ -860,10 +857,45 @@ function openColumnChooser() { }); } +function initCountAll() { + var button = document.querySelector(".count-all"); + if (!button) { + return; + } + button.addEventListener("click", async function () { + var count = document.querySelector(".table-count"); + var error = document.querySelector(".count-error"); + button.disabled = true; + button.textContent = "Counting…"; + error.textContent = ""; + try { + var response = await fetch(button.dataset.countUrl + location.search, { + method: "POST", + headers: { + Accept: "application/json", + }, + }); + var data = await response.json(); + if (!response.ok || !data.ok) { + throw new Error((data.errors || ["Count failed"]).join(" ")); + } + count.textContent = + data.count.toLocaleString("en-US") + + (data.count === 1 ? " row" : " rows"); + button.remove(); + } catch (ex) { + error.textContent = ex.message || "Count failed"; + button.disabled = false; + button.textContent = "count all"; + } + }); +} + // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; + initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/telemetry.py b/datasette/telemetry.py new file mode 100644 index 00000000..2fd5a9dc --- /dev/null +++ b/datasette/telemetry.py @@ -0,0 +1,481 @@ +""" +OpenTelemetry integration for Datasette. + +This uses `opentelemetry-api` only. Providers, exporters and sampling are +configured by whoever runs Datasette, for example `opentelemetry-instrument`. +""" + +import contextvars +import re +import threading +import time +import weakref +from contextlib import contextmanager + +from opentelemetry import context as otel_context_api +from opentelemetry import metrics as otel_metrics +from opentelemetry import trace as otel_trace +from opentelemetry.propagate import extract +from opentelemetry.propagators.textmap import Getter +from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span + +from .telemetry_registry import ( + DB_NAMESPACE, + DB_SYSTEM, + ERROR_TYPE, + HTTP_REQUEST_METHOD, + HTTP_RESPONSE_STATUS_CODE, + INTERNAL_CLIENT, + M_CONNECTIONS_OPEN, + M_OPERATION_DURATION, + M_QUERIES_INTERRUPTED, + M_QUERIES_PENDING, + M_THREADS_LIMIT, + M_THREADS_QUEUE_DEPTH, + M_WRITE_QUEUE_DEPTH, + M_WRITE_QUEUE_WAIT, + OPERATION, + SERVER_ADDRESS, + URL_PATH, + URL_SCHEME, + USER_AGENT_ORIGINAL, +) +from .version import __version__ + +# True while code is executing within a datasette.client request. Defined +# here rather than in app.py to avoid a circular import. +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + +# The semantic conventions version matching the attribute names used here. +# 1.30.0 renamed `db.system` to `db.system.name`, so update this when +# renaming attributes to match a newer version. +SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0" + +tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL) +meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL) + +MAX_SQL_LENGTH = 2048 + + +def sql_attribute(sql: str) -> str: + "Truncate SQL text so it is safe to attach to a span as an attribute." + sql = sql.strip() + if len(sql) <= MAX_SQL_LENGTH: + return sql + return sql[:MAX_SQL_LENGTH] + "…[truncated]" + + +def callback_name(fn) -> str: + """ + The name recorded as `datasette.callback` for a callback-style call. + + Falls back to the type name for callables such as `functools.partial` + that have no `__qualname__`. + """ + return getattr(fn, "__qualname__", type(fn).__name__) + + +def linked_root_span_kwargs(context=None): + """ + Keyword arguments that start a new root span with a ``Link`` back to + the current span. + + Use this for work that can outlive the span that caused it, such as a + background task or a ``block=False`` write. + + Pass ``context`` to link to the span in a previously captured context + instead of the current one. If there is no valid span, no link is added. + + Works with any tracer:: + + with my_tracer.start_as_current_span( + "myplugin.job", **linked_root_span_kwargs() + ): + ... + """ + cause = get_current_span(context).get_span_context() + links = [Link(cause)] if cause.is_valid else [] + return {"context": otel_context_api.Context(), "links": links} + + +# Keywords that can be recorded as db.operation.name. SQL can be supplied by +# users, so an allowlist keeps the number of distinct values small. +DB_OPERATION_ALLOWLIST = frozenset( + { + "SELECT", + "INSERT", + "UPDATE", + "DELETE", + "CREATE", + "DROP", + "ALTER", + "PRAGMA", + "EXPLAIN", + "REPLACE", + "VACUUM", + "ANALYZE", + "WITH", + } +) + +_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)") + + +def sql_operation_name(sql: str) -> str | None: + """ + The statement's leading keyword if it is in the allowlist, else None. + + Statements that start with a comment or "(" return None. Statements + starting with a CTE return `WITH`. Only call this for a single statement. + """ + match = _LEADING_KEYWORD.match(sql) + if not match: + return None + keyword = match.group(1).upper() + if keyword in DB_OPERATION_ALLOWLIST: + return keyword + return None + + +# --- The HTTP request span ------------------------------------------------ + + +class _ScopeHeadersGetter(Getter): + "Read W3C trace context from an ASGI scope's headers." + + def get(self, carrier, key): + wanted = key.lower().encode("latin-1") + values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted] + return values or None + + def keys(self, carrier): + return [k.decode("latin-1") for k, _ in carrier] + + +_HEADERS_GETTER = _ScopeHeadersGetter() + + +# Methods defined by RFC 9110 plus PATCH (RFC 5789). Anything else is +# recorded as `_OTHER`, as recommended by semantic conventions. +_KNOWN_METHODS = frozenset( + {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"} +) + + +def clamp_http_method(method): + "The request method if it is one we recognise, else ``_OTHER``." + method = (method or "").upper() + return method if method in _KNOWN_METHODS else "_OTHER" + + +def _first_header(headers, name): + "The first value of a header, decoded, or None." + for key, value in headers: + if key.lower() == name: + return value.decode("latin-1") + return None + + +def _url_path(scope): + """ + The request path, with any query string removed. + + Prefers `raw_path`, which preserves encoded slashes in database and + table names. Some clients include the query string in `raw_path`, so + that is stripped as well. + """ + raw_path = scope.get("raw_path") + if raw_path: + if isinstance(raw_path, bytes): + raw_path = raw_path.decode("latin-1") + return raw_path.split("?", 1)[0] + return scope.get("path", "") + + +# The request span is passed to the router in the ASGI scope, because a +# plugin's asgi_wrapper() middleware may have made its own span current. +# Absent if the span is not recording. +REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span" + + +def request_span(scope): + """ + The recording request span for an ASGI scope, or None. + + Falls back to the current span, for when Datasette is running under + other instrumentation. + """ + span = scope.get(REQUEST_SPAN_SCOPE_KEY) + if span is None: + span = otel_trace.get_current_span() + return span if span.is_recording() else None + + +class TelemetryMiddleware: + """ + One `SpanKind.SERVER` span per HTTP request. + + The span ends after the full response, including any streamed body, + has been sent. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + # Pass lifespan and websocket scopes straight through + if scope["type"] != "http": + await self.app(scope, receive, send) + return + headers = scope.get("headers") or [] + # Uses the global propagator, configured with OTEL_PROPAGATORS + context = extract(headers, getter=_HEADERS_GETTER) + method = clamp_http_method(scope.get("method", "")) + # Renamed to include the route once routing has happened + with tracer.start_as_current_span( + method, context=context, kind=SpanKind.SERVER + ) as span: + if not span.is_recording(): + # No provider installed, or the trace was not sampled + await self.app(scope, receive, send) + return + span.set_attribute(HTTP_REQUEST_METHOD, method) + span.set_attribute(URL_PATH, _url_path(scope)) + scheme = scope.get("scheme") + if scheme: + span.set_attribute(URL_SCHEME, scheme) + host = _first_header(headers, b"host") + if host: + span.set_attribute(SERVER_ADDRESS, host) + user_agent = _first_header(headers, b"user-agent") + if user_agent: + span.set_attribute(USER_AGENT_ORIGINAL, user_agent) + if _in_datasette_client.get(): + span.set_attribute(INTERNAL_CLIENT, True) + + scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span}) + + # Some responses are sent without a Response object, so the + # status is captured by wrapping send() + status_holder = {} + + async def wrapped_send(message): + if ( + message["type"] == "http.response.start" + and "status" not in status_holder + ): + status_holder["status"] = message["status"] + await send(message) + + escaped = False + try: + await self.app(scope, receive, wrapped_send) + except BaseException as exception: + # Includes asyncio.CancelledError when a client disconnects + escaped = True + span.set_attribute(ERROR_TYPE, type(exception).__name__) + span.set_status(Status(StatusCode.ERROR, str(exception))) + raise + finally: + status = status_holder.get("status") + if status is not None: + span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status) + # 4xx responses are not errors for a server span. If an + # exception escaped, keep its class name as error.type. + if status >= 500 and not escaped: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute(ERROR_TYPE, str(status)) + + +# --- Metrics -------------------------------------------------------------- + + +def _duration_attributes(database_name, operation): + return { + DB_SYSTEM: "sqlite", + DB_NAMESPACE: database_name, + OPERATION: operation, + } + + +# Instruments use plain text descriptions. The registry entries have longer +# reStructuredText descriptions for the documentation. + +sql_operation_duration = meter.create_histogram( + M_OPERATION_DURATION, + unit=M_OPERATION_DURATION.unit, + description="Duration of a SQL operation issued by Datasette", + explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets, +) + +write_queue_wait = meter.create_histogram( + M_WRITE_QUEUE_WAIT, + unit=M_WRITE_QUEUE_WAIT.unit, + description=( + "Time a write spent queued behind the single write thread for its database" + ), + explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets, +) + +queries_interrupted = meter.create_counter( + M_QUERIES_INTERRUPTED, + unit=M_QUERIES_INTERRUPTED.unit, + description="Queries cancelled for exceeding sql_time_limit_ms", +) + + +@contextmanager +def record_operation_duration(database_name, operation): + """ + Record `db.client.operation.duration` for one SQL operation. + + Sets `error.type` to the exception class on failure. For a `block=False` + write this measures the time taken to enqueue the write. + """ + attributes = _duration_attributes(database_name, operation) + started = time.perf_counter() + try: + yield + except BaseException as exception: + attributes[ERROR_TYPE] = type(exception).__qualname__ + raise + finally: + sql_operation_duration.record(time.perf_counter() - started, attributes) + + +def record_write_queue_wait(database_name, waited_ns): + write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name}) + + +def record_query_interrupted(database_name): + queries_interrupted.add(1, {DB_NAMESPACE: database_name}) + + +# Live Datasette instances reported by the gauges below. The lock is needed +# because gauge callbacks run on the SDK's collection thread. +# +# The pool gauges do not identify which instance they came from, so they +# are only meaningful for a process running a single Datasette instance. +_live_datasettes = weakref.WeakSet() +_live_datasettes_lock = threading.Lock() + + +def register_datasette(ds): + "Start reporting pool/queue gauges for this Datasette instance." + with _live_datasettes_lock: + _live_datasettes.add(ds) + + +def unregister_datasette(ds): + "Stop reporting gauges for an instance that has been closed." + with _live_datasettes_lock: + _live_datasettes.discard(ds) + + +def _live_instances(): + with _live_datasettes_lock: + return list(_live_datasettes) + + +def _databases_of(ds): + "Every Database attached to an instance, including the internal database." + databases = list(ds.databases.values()) + internal = getattr(ds, "_internal_database", None) + if internal is not None: + databases.append(internal) + return databases + + +def observe_sql_thread_limit(options=None): + "Size of the shared read-query thread pool (the num_sql_threads setting)." + for ds in _live_instances(): + if ds.executor is None: + # num_sql_threads=0 - queries run on the event loop, no pool. + continue + yield otel_metrics.Observation(ds.setting("num_sql_threads"), {}) + + +def observe_sql_thread_queue_depth(options=None): + """ + Read queries waiting for a free thread in the shared pool. + + `_work_queue` is a private attribute of ThreadPoolExecutor, so this + reports nothing if it is missing. + """ + for ds in _live_instances(): + if ds.executor is None: + continue + work_queue = getattr(ds.executor, "_work_queue", None) + if work_queue is None: + continue + yield otel_metrics.Observation(work_queue.qsize(), {}) + + +def observe_pending_queries(options=None): + """ + Read queries submitted to the pool and not yet finished, per database. + + Reads `len()` without `_pending_execute_futures_lock` to avoid blocking + queries. + """ + for ds in _live_instances(): + for db in _databases_of(ds): + yield otel_metrics.Observation( + len(db._pending_execute_futures), {DB_NAMESPACE: db.name} + ) + + +def observe_write_queue_depth(options=None): + "Writes queued behind the single write thread, per database." + for ds in _live_instances(): + for db in _databases_of(ds): + write_queue = db._write_queue + if write_queue is None: + # No write has ever been queued for this database. + continue + yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name}) + + +def observe_open_connections(options=None): + "Open SQLite connections tracked for closing, per database." + for ds in _live_instances(): + for db in _databases_of(ds): + yield otel_metrics.Observation( + len(db._all_connections), {DB_NAMESPACE: db.name} + ) + + +sql_thread_limit_gauge = meter.create_observable_gauge( + M_THREADS_LIMIT, + callbacks=[observe_sql_thread_limit], + unit=M_THREADS_LIMIT.unit, + description="Maximum concurrent read queries (the num_sql_threads setting)", +) + +sql_thread_queue_depth_gauge = meter.create_observable_gauge( + M_THREADS_QUEUE_DEPTH, + callbacks=[observe_sql_thread_queue_depth], + unit=M_THREADS_QUEUE_DEPTH.unit, + description="Read queries waiting for a free thread in the shared SQL pool", +) + +pending_queries_gauge = meter.create_observable_gauge( + M_QUERIES_PENDING, + callbacks=[observe_pending_queries], + unit=M_QUERIES_PENDING.unit, + description="Read queries submitted to the pool and not yet complete", +) + +write_queue_depth_gauge = meter.create_observable_gauge( + M_WRITE_QUEUE_DEPTH, + callbacks=[observe_write_queue_depth], + unit=M_WRITE_QUEUE_DEPTH.unit, + description="Writes queued behind a database's single write thread", +) + +open_connections_gauge = meter.create_observable_gauge( + M_CONNECTIONS_OPEN, + callbacks=[observe_open_connections], + unit=M_CONNECTIONS_OPEN.unit, + description="Open SQLite connections tracked for closing", +) diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py new file mode 100644 index 00000000..63d5b89b --- /dev/null +++ b/datasette/telemetry_registry.py @@ -0,0 +1,502 @@ +""" +Every span, metric and attribute that Datasette emits. + +These entries are used by the instrumentation code, by `docs/telemetry_doc.py` +to generate the documentation, and by `tests/test_telemetry_registry.py` to +check that the emitted telemetry matches the registry. +""" + +from opentelemetry.trace import SpanKind + + +class Attribute(str): + """ + A span attribute key, carrying its own documentation. + + Subclasses `str` so it can be handed straight to `set_attribute()`. + + Part of Datasette's public plugin API - plugins declare their own + telemetry registries with these classes. See the "Telemetry for plugin + authors" documentation. + """ + + __slots__ = ("description", "optional", "values") + + def __new__(cls, name, description, optional=False, values=None): + self = super().__new__(cls, name) + self.description = description + self.optional = optional + # The allowed values for this attribute, or None to allow any value + self.values = frozenset(values) if values is not None else None + return self + + def __reduce__(self): + # Copies and pickles become a plain str, since __new__ requires the + # extra arguments. ConsoleMetricExporter deepcopies attribute keys. + return (str, (str(self),)) + + def __repr__(self): + return f"Attribute({str(self)!r})" + + +class SpanName(str): + """A span name, carrying its documentation and the attributes it may set. + + Part of Datasette's public plugin API, like `Attribute`. + """ + + __slots__ = ("attributes", "description", "dynamic", "kind", "prefix") + + def __new__( + cls, + name, + description, + attributes=(), + prefix=False, + dynamic=False, + kind=SpanKind.INTERNAL, + ): + self = super().__new__(cls, name) + self.description = description + self.attributes = tuple(attributes) + # Match emitted names that start with this prefix, for names with a + # variable suffix such as SpanName("chat ", ..., prefix=True) + self.prefix = prefix + # The emitted name is built at runtime, so `span_for()` matches it by + # span kind. The entry's string is a template for the documentation. + self.dynamic = dynamic + self.kind = kind + return self + + def __reduce__(self): + # See Attribute.__reduce__. + return (str, (str(self),)) + + def __repr__(self): + return f"SpanName({str(self)!r})" + + +class MetricName(str): + "A metric name, carrying its instrument kind, unit and attributes." + + __slots__ = ("attributes", "buckets", "description", "kind", "unit") + + def __new__(cls, name, kind, unit, description, attributes=(), buckets=None): + self = super().__new__(cls, name) + self.kind = kind + self.unit = unit + self.description = description + self.attributes = tuple(attributes) + # Explicit bucket boundaries, for histograms only + self.buckets = tuple(buckets) if buckets is not None else None + return self + + def __reduce__(self): + # See Attribute.__reduce__. + return (str, (str(self),)) + + def __repr__(self): + return f"MetricName({str(self)!r})" + + +COUNTER = "Counter" +UPDOWN_COUNTER = "UpDownCounter" +HISTOGRAM = "Histogram" +GAUGE = "Observable gauge" + + +# --- Attributes ----------------------------------------------------------- + +HTTP_REQUEST_METHOD = Attribute( + "http.request.method", + "The HTTP request method. Methods outside the nine defined by RFC 9110 " + "and RFC 5789 are recorded as ``_OTHER``.", +) +HTTP_RESPONSE_STATUS_CODE = Attribute( + "http.response.status_code", + "The HTTP response status code. Omitted if no response was started.", + optional=True, +) +HTTP_ROUTE = Attribute( + "http.route", + "The regular expression for the matched route, for example " + "``/(?P[^\\/\\.]+)/(?P
      [^\\/\\.]+)(\\.(?P\\w+))?$`` " + "for a table page. Use this attribute to group requests by route. " + "Omitted when no route matches.", + optional=True, +) +URL_PATH = Attribute( + "url.path", + "The URL path, excluding the query string.", +) +URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.") +SERVER_ADDRESS = Attribute( + "server.address", + "The ``Host`` header, including any ``:port`` suffix. This value is " + "supplied by the client.", + optional=True, +) +USER_AGENT_ORIGINAL = Attribute( + "user_agent.original", + "The ``User-Agent`` header, verbatim. Omitted if the client sent none.", + optional=True, +) +INTERNAL_CLIENT = Attribute( + "datasette.internal_client", + "``True`` for requests made through ``datasette.client``. Calls made " + "inside another request produce a nested ``SERVER`` span. Filter on " + "this attribute to exclude internal requests from request counts. " + "Omitted for requests received over the network.", + optional=True, +) +ERROR_TYPE = Attribute( + "error.type", + "The exception class name for a failed operation. On HTTP spans, also " + "set to the status code as a string for 5xx responses. A 4xx response " + "alone does not set this attribute or an error status.", + optional=True, +) + +DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") +DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") +OPERATION = Attribute( + "datasette.operation", + "Whether the operation was a read or a write.", + values={"read", "write"}, +) +DB_QUERY_TEXT = Attribute( + "db.query.text", + "The SQL, truncated to 2048 characters. Bound parameter values are not " + "recorded. For callback methods, ``datasette.callback`` is recorded instead.", + optional=True, +) +CALLBACK = Attribute( + "datasette.callback", + "The qualified name of the Python callable passed to ``execute_fn()``, " + "``execute_write_fn()`` or ``execute_isolated_fn()``, for example " + "``TableInsertView.post..insert_or_upsert_rows``. Set instead of " + "``db.query.text``. Lambdas appear as ````; use a named function " + "for a more descriptive span.", + optional=True, +) +DB_OPERATION_NAME = Attribute( + "db.operation.name", + "The statement's leading keyword, such as ``SELECT``, ``INSERT`` or " + "``CREATE``, if it matches the supported allowlist. Statements beginning " + "with a common table expression report ``WITH``. Omitted for unrecognized " + "keywords and ``execute_write_script()``.", + optional=True, +) +PARAM_COUNT = Attribute( + "datasette.param_count", + "Number of bound parameters. Recorded instead of the values themselves.", + optional=True, +) +PARAM_SETS = Attribute( + "datasette.param_sets", + "Number of parameter sets consumed by ``execute_write_many()``. " + "The parameter values are not recorded.", + optional=True, +) +TIME_LIMIT_MS = Attribute( + "datasette.time_limit_ms", + "Time limit applied to the read query, in milliseconds: " + ":ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.", + optional=True, +) +ROWS_RETURNED = Attribute( + "datasette.rows_returned", + "Number of rows returned by a successful read query.", + optional=True, +) +TRUNCATED = Attribute( + "datasette.truncated", + "True if the result was cut short by :ref:`setting_max_returned_rows`.", + optional=True, +) +INTERRUPTED = Attribute( + "datasette.interrupted", + "True if the query exceeded its time limit. The span status is set to " + "``ERROR`` unless the caller used a ``custom_time_limit`` shorter than " + ":ref:`setting_sql_time_limit_ms`, in which case the status is left unset.", + optional=True, +) +SQL_ERROR_SUPPRESSED = Attribute( + "datasette.sql_error_suppressed", + "True for a non-timeout SQL error with ``log_sql_errors=False``. The " + "exception is still raised, but the span status is left unset.", + optional=True, +) +EXECUTESCRIPT = Attribute( + "datasette.executescript", + "True for ``execute_write_script()``, which runs multiple statements.", + optional=True, +) +EXECUTEMANY = Attribute( + "datasette.executemany", + "True for ``execute_write_many()``, which runs one statement against many " + "parameter sets.", + optional=True, +) +ISOLATED_CONNECTION = Attribute( + "datasette.isolated_connection", + "True if the write ran on its own connection rather than the shared write " + "connection.", +) +TRANSACTION = Attribute( + "datasette.transaction", + "False for statements such as ``VACUUM`` that cannot run inside a transaction.", +) + + +# --- Spans ---------------------------------------------------------------- + +HTTP_REQUEST = SpanName( + "{http.request.method} {http.route}", + "One span per HTTP request, containing spans from plugin middleware and " + "database operations. Named for the HTTP method and matched route, or " + "just the method if no route matches. Incoming ``traceparent`` headers " + "are extracted using the global propagator to continue the caller's " + "trace. Incoming ``baggage`` is not propagated into plugin or downstream " + "context in this release. Set ``OTEL_PROPAGATORS=none`` to disable " + "extraction. For public instances, strip trace context headers at your " + "proxy if callers should not supply trace context.", + ( + HTTP_REQUEST_METHOD, + HTTP_ROUTE, + URL_PATH, + URL_SCHEME, + SERVER_ADDRESS, + USER_AGENT_ORIGINAL, + HTTP_RESPONSE_STATUS_CODE, + ERROR_TYPE, + INTERNAL_CLIENT, + ), + dynamic=True, + kind=SpanKind.SERVER, +) + +DB_QUERY = SpanName( + "db.query", + "A SQL operation, including time spent queued for a worker thread. For " + "``block=False`` writes, the span ends after the write is queued. " + "Callback methods record ``datasette.callback`` in place of ``db.query.text``.", + ( + DB_SYSTEM, + DB_NAMESPACE, + DB_QUERY_TEXT, + CALLBACK, + DB_OPERATION_NAME, + PARAM_COUNT, + PARAM_SETS, + TIME_LIMIT_MS, + ROWS_RETURNED, + TRUNCATED, + INTERRUPTED, + SQL_ERROR_SUPPRESSED, + EXECUTESCRIPT, + EXECUTEMANY, + ), + kind=SpanKind.CLIENT, +) + +DB_QUERY_EXECUTE = SpanName( + "db.query.execute", + "The read executing inside a SQL worker thread. Child of ``db.query``; the " + "gap between the two is time spent waiting for a thread.", +) + +DB_WRITE_QUEUE_WAIT = SpanName( + "db.write.queue_wait", + "Time a write spent waiting in its database's write queue. For " + "``block=True``, this is a child of ``db.query``. For ``block=False``, " + "it is a root span linked to the span that queued the write, since the " + "write can outlive that request.", +) + +DB_WRITE_EXECUTE = SpanName( + "db.write.execute", + "The write executing on the write thread. For ``block=True``, this is " + "a child of ``db.query``. For ``block=False``, it is a root span linked " + "to the span that queued the write.", + (ISOLATED_CONNECTION, TRANSACTION), +) + +STARTUP = SpanName( + "datasette.startup", + "Startup work performed by ``invoke_startup()``, including registration " + "hooks, schema catalog updates, saved queries, column type configuration " + "and the ``startup`` hook. Runs during instance startup, either before " + "serving requests or as part of the first request.", +) + +SPANS = ( + HTTP_REQUEST, + DB_QUERY, + DB_QUERY_EXECUTE, + DB_WRITE_QUEUE_WAIT, + DB_WRITE_EXECUTE, + STARTUP, +) + + +def span_for(emitted_name, kind=None, spans=None): + """ + Resolve an emitted span name to its registry entry, or None. + + Exact matches take precedence over `prefix=True` entries, which take + precedence over `dynamic=True` entries matched by `kind`. + + `spans` defaults to Datasette's own registry. + """ + if spans is None: + spans = SPANS + for span in spans: + if span.dynamic: + continue + if emitted_name == span: + return span + for span in spans: + if span.prefix and emitted_name.startswith(span): + return span + if kind is not None: + for span in spans: + if span.dynamic and span.kind == kind: + return span + return None + + +def metric_for(emitted_name, metrics=None): + """ + Resolve an emitted metric name to its registry entry, or None. + + `metrics` defaults to Datasette's own registry. + """ + if metrics is None: + metrics = METRICS + for metric in metrics: + if emitted_name == metric: + return metric + return None + + +def attribute_allowed(entry, emitted_key): + """ + Whether `emitted_key` is a registered attribute of `entry`. + + `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`. + """ + if entry is None: + return False + return emitted_key in entry.attributes + + +def attribute_value_allowed(entry, emitted_key, value): + """ + Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName` + or a `MetricName`). + + Any value is allowed if the attribute does not declare `values=`. + """ + if entry is None: + return False + for attribute in entry.attributes: + if attribute == emitted_key: + return attribute.values is None or value in attribute.values + return False + + +# --- Metrics -------------------------------------------------------------- + +# Bucket boundaries in seconds for every duration histogram. OpenTelemetry's +# defaults are designed for milliseconds and would put almost every SQLite +# query in the first bucket. These are the semantic conventions' recommended +# boundaries for db.client.operation.duration, plus 0.0001 and 0.0005 for +# fast in-process SQLite queries. +DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10) + +M_OPERATION_DURATION = MetricName( + "db.client.operation.duration", + HISTOGRAM, + "s", + "Duration of a SQL operation, including callback-based calls such as " + "``execute_fn()``. For ``block=False`` writes, measures enqueue time.", + (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE), + buckets=DURATION_BUCKETS, +) + +M_WRITE_QUEUE_WAIT = MetricName( + "datasette.write.queue_wait", + HISTOGRAM, + "s", + "Time each write waited in its database's write queue.", + (DB_NAMESPACE,), + buckets=DURATION_BUCKETS, +) + +M_QUERIES_INTERRUPTED = MetricName( + "datasette.sql.queries.interrupted", + COUNTER, + "{query}", + "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A " + "rising rate can indicate that queries need optimization or a higher " + "time limit. Caller-selected timeouts shorter than this limit, such as " + "those used for facet suggestion, are excluded.", + (DB_NAMESPACE,), +) + +M_THREADS_LIMIT = MetricName( + "datasette.sql.threads.limit", + GAUGE, + "{thread}", + "Maximum concurrent read queries, configured by " + ":ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` " + "is ``0``.", +) + +M_THREADS_QUEUE_DEPTH = MetricName( + "datasette.sql.threads.queue_depth", + GAUGE, + "{query}", + "Read queries waiting for a free SQL thread. Sustained values above " + "zero indicate a saturated read pool.", +) + +M_QUERIES_PENDING = MetricName( + "datasette.sql.queries.pending", + GAUGE, + "{query}", + "Read queries submitted to the pool and not yet complete. Sum across " + "databases and compare with ``datasette.sql.threads.limit`` to assess " + "pool usage.", + (DB_NAMESPACE,), +) + +M_WRITE_QUEUE_DEPTH = MetricName( + "datasette.write.queue_depth", + GAUGE, + "{write}", + "Writes waiting for a database's single write thread. Increasing " + "``num_sql_threads`` does not increase write concurrency. Not reported for " + "databases that have never been written to.", + (DB_NAMESPACE,), +) + +M_CONNECTIONS_OPEN = MetricName( + "datasette.connections.open", + GAUGE, + "{connection}", + "Open SQLite connections managed by Datasette.", + (DB_NAMESPACE,), +) + +METRICS = ( + M_OPERATION_DURATION, + M_WRITE_QUEUE_WAIT, + M_QUERIES_INTERRUPTED, + M_THREADS_LIMIT, + M_THREADS_QUEUE_DEPTH, + M_QUERIES_PENDING, + M_WRITE_QUEUE_DEPTH, + M_CONNECTIONS_OPEN, +) diff --git a/datasette/telemetry_testing.py b/datasette/telemetry_testing.py new file mode 100644 index 00000000..77a4431b --- /dev/null +++ b/datasette/telemetry_testing.py @@ -0,0 +1,427 @@ +""" +Pytest helpers for testing OpenTelemetry instrumentation - Datasette's own +and any plugin's. Part of Datasette's public plugin API; see the "Telemetry +for plugin authors" documentation. + +Usage from a plugin's ``conftest.py``:: + + from datasette.telemetry_testing import ( # noqa: F401 + MetricsCollector, + otel_metrics, + otel_meter_provider, + otel_provider, + otel_spans, + ) + +Tests can then use the ``otel_spans`` and ``otel_metrics`` fixtures. The +OpenTelemetry SDK is imported lazily, and the fixtures skip if it is not +installed. +""" + +import subprocess +import sys + +import pytest + +from .telemetry_registry import ( + attribute_allowed, + attribute_value_allowed, + metric_for, + span_for, +) + +_span_exporter = None +_metric_reader = None + + +def install_span_exporter(): + """ + Install a TracerProvider + InMemorySpanExporter once per process and + return the exporter, or None when the SDK is not installed. + + Uses `SimpleSpanProcessor` so spans are exported as soon as they end. + """ + global _span_exporter + if _span_exporter is not None: + return _span_exporter + try: + from opentelemetry import trace as otel_trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + except ImportError: + return None + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel_trace.set_tracer_provider(provider) + # set_tracer_provider() is ignored if a provider was already installed, + # in which case the fixtures skip + if otel_trace.get_tracer_provider() is not provider: + return None + _span_exporter = exporter + return exporter + + +def install_metric_reader(): + """ + Install a MeterProvider + InMemoryMetricReader once per process and + return the reader, or None when the SDK is not installed. + + Uses delta temporality for counters and histograms, so each collection + only reports measurements since the previous one. + """ + global _metric_reader + if _metric_reader is not None: + return _metric_reader + try: + from opentelemetry import metrics as otel_metrics_api + from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + InMemoryMetricReader, + ) + except ImportError: + return None + reader = InMemoryMetricReader( + preferred_temporality={ + Counter: AggregationTemporality.DELTA, + Histogram: AggregationTemporality.DELTA, + } + ) + provider = MeterProvider(metric_readers=[reader]) + otel_metrics_api.set_meter_provider(provider) + if otel_metrics_api.get_meter_provider() is not provider: + return None + _metric_reader = reader + return reader + + +@pytest.fixture(scope="session", autouse=True) +def otel_provider(): + "Install the span exporter once per test session, before any spans are created." + install_span_exporter() + + +@pytest.fixture(scope="session", autouse=True) +def otel_meter_provider(): + "Install the metric reader once per test session." + install_metric_reader() + + +@pytest.fixture(autouse=True) +def otel_reset(): + "Clear recorded spans and drain collected metrics after every test." + yield + if _span_exporter is not None: + _span_exporter.clear() + if _metric_reader is not None: + _metric_reader.get_metrics_data() + + +@pytest.fixture +def otel_spans(): + """ + The in-memory span exporter, cleared before the test. Call + `.get_finished_spans()` to retrieve spans. + """ + pytest.importorskip("opentelemetry.sdk") + exporter = install_span_exporter() + if exporter is None: + pytest.skip("OpenTelemetry SDK provider was not installed") + exporter.clear() + yield exporter + + +class MetricsCollector: + """ + Wraps an `InMemoryMetricReader`. + + `collect()` runs a collection cycle and stores a snapshot, which + `points()` and `point()` then query. + """ + + def __init__(self, reader): + self.reader = reader + self.snapshot = {} + # (instrumentation scope name, sdk Metric) pairs from the last collect() + self.collected = [] + + def collect(self): + self.snapshot = {} + self.collected = [] + data = self.reader.get_metrics_data() + if data is None: + return self.snapshot + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + scope_name = scope_metrics.scope.name if scope_metrics.scope else None + for metric in scope_metrics.metrics: + self.snapshot.setdefault(metric.name, []).extend( + metric.data.data_points + ) + self.collected.append((scope_name, metric)) + return self.snapshot + + def points(self, name, attributes=None): + "Data points for `name` whose attributes are a superset of `attributes`." + found = [] + for point in self.snapshot.get(name, []): + point_attributes = dict(point.attributes or {}) + if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()): + found.append(point) + return found + + def point(self, name, attributes=None): + "The single matching data point, asserting there is exactly one." + found = self.points(name, attributes) + assert len(found) == 1, ( + f"expected exactly one {name} point matching {attributes}, " + f"got {len(found)}: {found}" + ) + return found[0] + + +@pytest.fixture +def otel_metrics(): + "A `MetricsCollector`, drained before the test so counts start from zero." + pytest.importorskip("opentelemetry.sdk") + reader = install_metric_reader() + if reader is None: + pytest.skip("OpenTelemetry SDK meter provider was not installed") + reader.get_metrics_data() + yield MetricsCollector(reader) + + +def _scoped(finished_spans, scope_name): + if scope_name is None: + return list(finished_spans) + return [ + span + for span in finished_spans + if span.instrumentation_scope and span.instrumentation_scope.name == scope_name + ] + + +def assert_spans_conform(registry_spans, finished_spans, scope_name=None): + """ + Assert every finished span is registered in `registry_spans`, sets only + registered attributes and uses allowed attribute values. + + Pass `scope_name` to only check spans from that instrumentation scope. + """ + problems = [] + for span in _scoped(finished_spans, scope_name): + entry = span_for(str(span.name), kind=span.kind, spans=registry_spans) + if entry is None: + problems.append(f"unregistered span: {span.name!r}") + continue + for key, value in (span.attributes or {}).items(): + if not attribute_allowed(entry, str(key)): + problems.append(f"{span.name}: unregistered attribute {key!r}") + elif not attribute_value_allowed(entry, str(key), value): + problems.append( + f"{span.name}: {key}={value!r} not in the declared enum" + ) + assert not problems, "\n".join(problems) + + +def assert_spans_covered(registry_spans, finished_spans, scope_name=None): + """ + Assert every entry in `registry_spans` was emitted at least once, with + each of its attributes that is not `optional=True`. + """ + spans = _scoped(finished_spans, scope_name) + seen_attributes = {} + for span in spans: + entry = span_for(str(span.name), kind=span.kind, spans=registry_spans) + if entry is not None: + seen = seen_attributes.setdefault(str(entry), set()) + seen.update(str(key) for key in (span.attributes or {})) + problems = [] + for entry in registry_spans: + if str(entry) not in seen_attributes: + problems.append(f"registered span never emitted: {entry!r}") + continue + required = { + str(attribute) for attribute in entry.attributes if not attribute.optional + } + missing = required - seen_attributes[str(entry)] + if missing: + problems.append( + f"{entry}: registered attributes never emitted: {sorted(missing)}" + ) + assert not problems, "\n".join(problems) + + +# Registry instrument kinds mapped to the SDK data type collected for them. +# Both counter kinds collect as Sum, distinguished by is_monotonic. +_KIND_TO_DATA_TYPE = { + "Counter": "Sum", + "UpDownCounter": "Sum", + "Histogram": "Histogram", + "Observable gauge": "Gauge", +} +_KIND_IS_MONOTONIC = {"Counter": True, "UpDownCounter": False} + + +def _scoped_metrics(collector, scope_name): + for scope, metric in collector.collected: + if scope_name is None or scope == scope_name: + yield metric + + +def assert_metrics_conform(registry_metrics, collector, scope_name=None): + """ + Assert every metric in the collector's last `collect()` is registered in + `registry_metrics` with a matching instrument kind and unit, sets only + registered attributes and uses allowed attribute values. + + Pass `scope_name` to only check metrics from that instrumentation scope. + """ + problems = set() + for metric in _scoped_metrics(collector, scope_name): + entry = metric_for(metric.name, metrics=registry_metrics) + if entry is None: + problems.add(f"unregistered metric: {metric.name!r}") + continue + expected_data_type = _KIND_TO_DATA_TYPE.get(entry.kind) + actual_data_type = type(metric.data).__name__ + if expected_data_type is not None and actual_data_type != expected_data_type: + problems.add( + f"{metric.name}: registry declares {entry.kind}, " + f"SDK collected {actual_data_type}" + ) + expected_monotonic = _KIND_IS_MONOTONIC.get(entry.kind) + actual_monotonic = getattr(metric.data, "is_monotonic", None) + if ( + expected_monotonic is not None + and actual_monotonic is not None + and actual_monotonic != expected_monotonic + ): + problems.add( + f"{metric.name}: registry declares {entry.kind}, but the " + f"collected Sum is_monotonic={actual_monotonic}" + ) + if (metric.unit or "") != (entry.unit or ""): + problems.add( + f"{metric.name}: instrument unit {metric.unit!r} != " + f"registry unit {entry.unit!r}" + ) + for point in metric.data.data_points: + for key, value in dict(point.attributes or {}).items(): + if not attribute_allowed(entry, str(key)): + problems.add(f"{metric.name}: unregistered attribute {key!r}") + elif not attribute_value_allowed(entry, str(key), value): + problems.add( + f"{metric.name}: {key}={value!r} not in the declared enum" + ) + assert not problems, "\n".join(sorted(problems)) + + +def assert_metrics_covered(registry_metrics, collector, scope_name=None): + """ + Assert every entry in `registry_metrics` was collected at least once, + with each of its attributes that is not `optional=True`. + + Call `collect()` once after the workload and before this check. + """ + seen_attributes = {} + for metric in _scoped_metrics(collector, scope_name): + entry = metric_for(metric.name, metrics=registry_metrics) + if entry is None: + continue + seen = seen_attributes.setdefault(str(entry), set()) + for point in metric.data.data_points: + seen.update(str(key) for key in dict(point.attributes or {})) + problems = [] + for entry in registry_metrics: + if str(entry) not in seen_attributes: + problems.append(f"registered metric never collected: {entry!r}") + continue + required = { + str(attribute) for attribute in entry.attributes if not attribute.optional + } + missing = required - seen_attributes[str(entry)] + if missing: + problems.append( + f"{entry}: registered attributes never collected: {sorted(missing)}" + ) + assert not problems, "\n".join(problems) + + +def assert_no_forbidden_values( + forbidden, finished_spans=None, collector=None, scope_name=None +): + """ + Assert that none of the `forbidden` strings appear anywhere in the + emitted telemetry: span names, span attribute values, span event names + and attributes, span status descriptions, or metric point attributes. + + Use fake private values such as tokens or email addresses in your test + workload, then check that they were not recorded: + + FORBIDDEN = {"secret-token-123", "alice@example.com"} + run_workload_using_those_values() + assert_no_forbidden_values( + FORBIDDEN, + finished_spans=otel_spans.get_finished_spans(), + collector=otel_metrics, + ) + + Matches substrings of each value's string form. Empty strings in + `forbidden` are ignored. Leave `scope_name` unset to also check + Datasette's own telemetry. + """ + needles = [needle for needle in forbidden if needle] + leaks = set() + + def check(value, where): + text = str(value) + for needle in needles: + if needle in text: + leaks.add(f"{where} contains {needle!r}") + + if finished_spans is not None: + for span in _scoped(finished_spans, scope_name): + check(span.name, f"span name {str(span.name)!r}") + for key, value in (span.attributes or {}).items(): + check(value, f"{span.name} attribute {key}") + for event in span.events or (): + check(event.name, f"{span.name} event name") + for key, value in (event.attributes or {}).items(): + check(value, f"{span.name} event {event.name} attribute {key}") + if span.status is not None and span.status.description: + check(span.status.description, f"{span.name} status description") + if collector is not None: + for metric in _scoped_metrics(collector, scope_name): + for point in metric.data.data_points: + for key, value in dict(point.attributes or {}).items(): + check(value, f"metric {metric.name} attribute {key}") + assert not leaks, "forbidden values leaked into telemetry:\n" + "\n".join( + sorted(leaks) + ) + + +def assert_package_never_imports_sdk(*module_names): + """ + Import the named modules in a fresh interpreter and assert none of them + imported `opentelemetry.sdk`. + + Run the test that calls this early in your suite: on macOS with CPython + 3.13, starting a subprocess from a process with many threads can crash. + """ + imports = "; ".join(f"import {name}" for name in module_names) + code = ( + f"import sys; {imports}; " + "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert result.stdout.strip() == "[]", ( + f"importing {module_names} pulled in the OpenTelemetry SDK: " + f"{result.stdout.strip()}" + ) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 4927cb8d..32686af1 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,7 +3,6 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} - {% endblock %} {% block content %} @@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 18288439..e5aa46f3 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -8,6 +8,7 @@ {% endfor %} + {% for url in extra_js_urls %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 80249d9c..c73cdfb7 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,7 +3,6 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -198,7 +197,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } function displayError(data) { @@ -208,7 +207,7 @@ function displayError(data) { resultsContent.innerHTML = `
      Error: ${escapeHtml(data.error || 'Unknown error')}
      `; - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index b9fc636a..c0081c66 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,7 +3,6 @@ {% block title %}Explain a permission decision{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %}