diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index cf9b25a7..3fc83438 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,46 +14,24 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Check deployment prerequisites - id: deployment-prerequisites - env: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} - run: | - missing=() - for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do - if [[ -z "${!variable:-}" ]]; then - missing+=("$variable") - fi - done - if (( ${#missing[@]} )); then - echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" - echo "available=false" >> "$GITHUB_OUTPUT" - else - echo "available=true" >> "$GITHUB_OUTPUT" - fi - name: Check out datasette - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -61,18 +39,14 @@ jobs: fixtures-metadata.json \ plugins \ --extra-db-filename extra_database.db - # Package the config with the plugins, excluding test-only plugin secrets - # that reference temporary files outside the deployed container. - jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json - name: Build docs.db - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -84,7 +58,6 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Upload latest documentation database to S3 (only for main) - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - name: Deploy to docs as well (only for main) + if: ${{ github.ref == 'refs/heads/main' }} run: |- - # Keep development documentation separate from the stable release database. - s3-credentials put-object datasette-docs latest/docs.db docs.db \ - --content-type application/octet-stream + # Deploy docs.db to a different service + datasette publish cloudrun docs.db \ + --branch=$GITHUB_SHA \ + --version-note=$GITHUB_SHA \ + --extra-options="--setting template_debug 1" \ + --service=datasette-docs-latest diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 85369f6c..f5b8dbf6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,15 +2,9 @@ name: Playwright on: push: - branches: - - main pull_request: workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - permissions: contents: read diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index fa7ec6aa..d92ab82b 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -1,15 +1,6 @@ name: Check JavaScript for conformance with Prettier -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push] permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 232a34c7..21ed4c12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [published] + types: [created] permissions: contents: read @@ -51,8 +51,6 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 - # After the first non-prerelease 1.0 release, disable this job on 0.65.x, - # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -68,20 +66,26 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Upload stable documentation database to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Deploy stable-docs.datasette.io to Cloud Run run: |- - s3-credentials put-object datasette-docs docs.db docs.db \ - --content-type application/octet-stream + gcloud config set run/region us-central1 + gcloud config set project datasette-222320 + datasette publish cloudrun docs.db \ + --service=datasette-docs-stable deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index aa35338f..58635025 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -1,15 +1,6 @@ name: Check spelling in documentation -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 00000000..e9bd4bab --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,40 @@ +name: Calculate test coverage + +on: + push: + branches: + - main + pull_request: + branches: + - main +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out datasette + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . --group dev + python -m pip install pytest-cov + - name: Run tests + run: |- + ls -lah + cat .coveragerc + pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x + ls -lah + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 449855f3..5e81ed82 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -2,15 +2,9 @@ name: Test in Pyodide with shot-scraper on: push: - branches: - - main pull_request: workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - permissions: contents: read diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 700f3cce..2fdb3a40 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -1,15 +1,6 @@ name: Test SQLite versions -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read @@ -21,10 +12,10 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: ["3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] sqlite-version: [ #"3", # latest version - #"3.46", + "3.46", #"3.45", #"3.27", #"3.26", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8176a630..751eedfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,6 @@ name: Test -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read @@ -20,20 +11,16 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] - include: - - python-version: "3.14" - coverage: true + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) @@ -41,27 +28,12 @@ jobs: run: | pip install . --group dev pip freeze - - name: Install pytest-cov - if: ${{ matrix.coverage }} - run: pip install pytest-cov - name: Run tests run: | - if [ "${{ matrix.coverage }}" = "true" ]; then - COV="--cov=datasette --cov-config=.coveragerc" - pytest -n auto -m "not serial" $COV --cov-report= - pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term - else - pytest -n auto -m "not serial" - pytest -m "serial" - fi + pytest -n auto -m "not serial" + pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Upload coverage report - if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - name: Black run: | black --version diff --git a/Dockerfile b/Dockerfile index 58287dd7..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm AS build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/Justfile b/Justfile index d1b69378..6ffff870 100644 --- a/Justfile +++ b/Justfile @@ -49,18 +49,13 @@ export DATASETTE_SECRET := "not_a_secret" uv run cog -r README.md docs/*.rst # Serve live docs on localhost:8000 -@docs: shots cog blacken-docs +@docs: cog blacken-docs uv run make -C docs livehtml # Build docs as static HTML @docs-build: cog blacken-docs rm -rf docs/_build && cd docs && uv run make html -# Take any missing documentation screenshots defined in docs/shots.yml -@shots: - uv run --group shots shot-scraper install - cd docs && uv run --group shots shot-scraper multi shots.yml --no-clobber --reduced-motion --retina - # Apply Black @black: uv run black datasette tests diff --git a/README.md b/README.md index 1f79778f..393e8e5c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/datasette/__init__.py b/datasette/__init__.py index 982dcc79..e0022178 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,7 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor # noqa from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa from datasette.utils.asgi import ( # noqa Forbidden, diff --git a/datasette/app.py b/datasette/app.py index 3e4c5acf..c82ea075 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import contextvars from collections.abc import Iterable, Sequence from typing import TYPE_CHECKING, Any @@ -27,7 +28,7 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx2 +import httpx from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, @@ -41,7 +42,6 @@ from jinja2.exceptions import TemplateNotFound from markupsafe import Markup, escape from . import stored_queries, write_sql -from .background_tasks import BackgroundTask, BackgroundTaskSupervisor from .column_types import SQLiteType from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted @@ -49,16 +49,6 @@ from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource -from .telemetry import ( - TelemetryMiddleware, - _in_datasette_client, - clamp_http_method, - register_datasette, - request_span, - tracer, - unregister_datasette, -) -from .telemetry_registry import HTTP_ROUTE, STARTUP from .tokens import TokenInvalid from .tracer import AsgiTracer from .url_builder import Urls @@ -155,7 +145,6 @@ from .views.stored_queries import ( ) from .views.table import ( TableAutocompleteView, - TableCountView, TableDropView, TableFragmentView, TableInsertView, @@ -175,7 +164,8 @@ app_root = Path(__file__).parent.parent logger = logging.getLogger(__name__) -# _in_datasette_client is defined in telemetry.py to avoid a circular import +# Context variable to track when code is executing within a datasette.client request +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) class _DatasetteClientContext: @@ -325,7 +315,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action.name, parent, action.normalize_child(child)) + return (actor_key, action, parent, child) async def favicon(request, send): @@ -432,7 +422,6 @@ class Datasette: default_deny=False, ): self._startup_invoked = False - self._shutdown_invoked = False self._closed = False assert config_dir is None or isinstance( config_dir, Path @@ -464,11 +453,8 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this - self._setup_db_done = False - self._suppress_background_tasks = False try: self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -476,10 +462,8 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() else: raise - self._background_tasks = BackgroundTaskSupervisor(self) self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: @@ -651,8 +635,6 @@ class Datasette: self.root_enabled = False self.default_deny = default_deny self.client = DatasetteClient(self) - # Last, so metric callbacks never see a partially initialized instance - register_datasette(self) async def apply_metadata_json(self): # Apply any metadata entries from metadata.json to the internal tables @@ -793,61 +775,57 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # Group spans created during startup under a single parent span - with tracer.start_as_current_span(STARTUP): - # Register event classes - event_classes = [] - for hook in pm.hook.register_events(datasette=self): - extra_classes = await await_me_maybe(hook) - if extra_classes: - event_classes.extend(extra_classes) - self.event_classes = tuple(event_classes) + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): - if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): - raise StartupError(f"Duplicate action name: {action.name}") - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action + # Register actions, but watch out for duplicate name/abbr + action_names = {} + action_abbrs = {} + for hook in pm.hook.register_actions(datasette=self): + if hook: + for action in hook: + if ( + action.name in action_names + and action != action_names[action.name] + ): + raise StartupError(f"Duplicate action name: {action.name}") + if ( + action.abbr + and action.abbr in action_abbrs + and action != action_abbrs[action.abbr] + ): + raise StartupError(f"Duplicate action abbr: {action.abbr}") + action_names[action.name] = action + if action.abbr: + action_abbrs[action.abbr] = action + self.actions[action.name] = action - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError( - f"Duplicate column type name: {ct_cls.name}" - ) - self._column_types[ct_cls.name] = ct_cls + # Register column types (classes, not instances) + self._column_types = {} + for hook in pm.hook.register_column_types(datasette=self): + if hook: + for ct_cls in hook: + if ct_cls.name in self._column_types: + raise StartupError(f"Duplicate column type name: {ct_cls.name}") + self._column_types[ct_cls.name] = ct_cls - for hook in pm.hook.prepare_jinja2_environment( - env=self._jinja_env, datasette=self - ): - await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() - for hook in pm.hook.startup(datasette=self): - await await_me_maybe(hook) - self._startup_invoked = True + for hook in pm.hook.prepare_jinja2_environment( + env=self._jinja_env, datasette=self + ): + await await_me_maybe(hook) + # Ensure internal tables and metadata are populated before startup hooks + await self._refresh_schemas() + await self._save_queries_from_config() + # Load column_types from config into internal DB + await self._apply_column_types_config() + for hook in pm.hook.startup(datasette=self): + await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -980,8 +958,6 @@ class Datasette: if self._closed: return self._closed = True - # Stop reporting metrics before closing databases - unregister_datasette(self) first_exception = None dbs = list(self.databases.values()) + [self._internal_database] for db in dbs: @@ -1553,28 +1529,15 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: - # Extension loading is only enabled for as long as it takes to - # load the configured extensions. Leaving it enabled would let - # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - try: - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - if sys.version_info >= (3, 12): - conn.load_extension(path, entrypoint=entrypoint) - else: - # Connection.load_extension() only gained the - # entrypoint argument in Python 3.12 - conn.execute( - "SELECT load_extension(?, ?)", [path, entrypoint] - ) - else: - conn.load_extension(extension) - finally: - conn.enable_load_extension(False) + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1767,145 +1730,8 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) - if action == "view-table": - sql, params = await self._apply_derived_table_permissions_to_sql( - sql, - params, - actor=actor, - parent=parent, - include_is_private=include_is_private, - ) return ResourcesSQL(sql, params) - async def _allowed_derived_table_source( - self, database, source, *, actor, dependencies - ): - """Check an immediate source, denying sources that are themselves derived.""" - if any( - TableResource.normalize_child(table) - == TableResource.normalize_child(source) - for table in dependencies - ): - return False - # The source has no dependency in this map. Evaluate its own permission - # and prerequisites without starting another dependency check. - verdicts = await self._allowed_many( - actions=["view-table"], - resource=TableResource(database, source), - actor=actor, - check_derived=False, - ) - return verdicts["view-table"] - - async def _apply_derived_table_permissions_to_sql( - self, - sql, - params, - *, - actor, - parent, - include_is_private, - ): - databases = ( - [(parent, self.databases[parent])] - if parent in self.databases - else ([] if parent is not None else list(self.databases.items())) - ) - dependency_maps = dict( - zip( - (name for name, _ in databases), - await asyncio.gather( - *(db.derived_table_dependencies() for _, db in databases) - ), - ) - ) - dependencies = [ - (database_name, child, source) - for database_name, dependency_map in dependency_maps.items() - for child, source in dependency_map.items() - ] - if not dependencies: - return sql, params - - sources = sorted( - {(database_name, source) for database_name, _, source in dependencies} - ) - actor_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=actor, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - actor_allowed = dict(zip(sources, actor_verdicts)) - - anonymous_allowed = {} - if include_is_private: - anonymous_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=None, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - anonymous_allowed = dict(zip(sources, anonymous_verdicts)) - - wrapped_params = dict(params) - derived_rows = [ - [ - database_name, - child, - int(actor_allowed[(database_name, source)]), - *( - [int(anonymous_allowed[(database_name, source)])] - if include_is_private - else [] - ), - ] - for database_name, child, source in dependencies - ] - derived_param = "_datasette_derived_permissions" - while derived_param in wrapped_params: - derived_param += "_" - wrapped_params[derived_param] = json.dumps(derived_rows) - - derived_columns = "parent, child, source_allowed" - select_columns = "allowed.parent, allowed.child, allowed.reason" - if include_is_private: - derived_columns += ", source_anonymous_allowed" - select_columns += ( - ", CASE WHEN derived.source_anonymous_allowed = 0 " - "THEN 1 ELSE allowed.is_private END AS is_private" - ) - wrapped_sql = f""" -WITH derived_permissions({derived_columns}) AS ( - SELECT - json_extract(value, '$[0]'), - json_extract(value, '$[1]'), - json_extract(value, '$[2]') - {", json_extract(value, '$[3]')" if include_is_private else ""} - FROM json_each(:{derived_param}) -), -allowed AS ( -{sql} -) -SELECT {select_columns} -FROM allowed -LEFT JOIN derived_permissions AS derived - ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE -WHERE COALESCE(derived.source_allowed, 1) = 1 -ORDER BY allowed.parent, allowed.child -""".strip() - return wrapped_sql, wrapped_params - async def allowed_resources( self, action: str, @@ -2108,12 +1934,6 @@ ORDER BY allowed.parent, allowed.child ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ - return await self._allowed_many( - actions=actions, resource=resource, actor=actor, check_derived=True - ) - - async def _allowed_many(self, *, actions, resource, actor, check_derived): - """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, @@ -2151,7 +1971,7 @@ ORDER BY allowed.parent, allowed.child to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, self.actions[name], parent, child) + key = _permission_cache_key(actor, name, parent, child) if key in cache: final[name] = cache[key] continue @@ -2167,28 +1987,6 @@ ORDER BY allowed.parent, allowed.child child=child, ) - if ( - check_derived - and "view-table" in to_check - and raw.get("view-table") - and isinstance(resource, TableResource) - and parent in self.databases - ): - dependencies = await self.databases[parent].derived_table_dependencies() - source = next( - ( - source - for table, source in dependencies.items() - if TableResource.normalize_child(table) - == TableResource.normalize_child(child) - ), - None, - ) - if source is not None: - raw["view-table"] = await self._allowed_derived_table_source( - parent, source, actor=actor, dependencies=dependencies - ) - def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2206,9 +2004,7 @@ ORDER BY allowed.parent, allowed.child # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[ - _permission_cache_key(actor, self.actions[name], parent, child) - ] = final[name] + cache[_permission_cache_key(actor, name, parent, child)] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2299,18 +2095,6 @@ ORDER BY allowed.parent, allowed.child from datasette.resources import TableResource other_table = fk["other_table"] - # Foreign key declarations can spell the target with different casing. - target_table = ( - await db.execute( - "select name from sqlite_master where type='table' and name=? collate nocase", - [other_table], - ) - ).first() - if target_table is None: - # SQLite accepts a foreign key to a table that does not exist, and - # linking to it would only lead to a 404 - return {} - other_table = target_table[0] other_column = fk["other_column"] if other_column is None: other_pks = await db.primary_keys(other_table) @@ -2494,21 +2278,6 @@ ORDER BY allowed.parent, allowed.child ) return d - def _tasks(self): - return { - "tasks": [ - { - "name": t.name, - "state": t.state, - "function": t.function, - "started_at": t.started_at, - "exception": repr(t.exception) if t.exception else None, - } - for t in self._background_tasks.tasks() - ], - "launched": self._background_tasks.launched, - } - def _actor(self, request): return {"actor": request.actor} @@ -2615,8 +2384,6 @@ ORDER BY allowed.parent, allowed.child datasette=self, ): extra_vars = await await_me_maybe(extra_vars) - if extra_vars is None: - continue assert isinstance( extra_vars, dict ), f"extra_vars is of type {type(extra_vars)}" @@ -2679,7 +2446,7 @@ ORDER BY allowed.parent, allowed.child ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + expire_after + expires_at = int(time.time()) + (24 * 60 * 60) data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2799,12 +2566,6 @@ ORDER BY allowed.parent, allowed.child ), r"/-/threads(\.(?Pjson))?$", ) - add_route( - JsonDataView.as_view( - self, "tasks.json", self._tasks, permission="permissions-debug" - ), - r"/-/tasks(\.(?Pjson))?$", - ) add_route( JsonDataView.as_view( self, @@ -2979,10 +2740,6 @@ ORDER BY allowed.parent, allowed.child TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) - add_route( - TableCountView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", - ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -3046,130 +2803,26 @@ ORDER BY allowed.parent, allowed.child raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) - async def _startup_sequence(self): - """Idempotently run the full startup sequence: table counts for - immutable databases, then invoke_startup(). Safe to call more than - once and safe to call concurrently - callers block until whichever - call got there first has finished. - - This is the single entry point used by both AsgiLifespan (so - real deployments finish startup before accepting requests) and - AsgiRunOnFirstRequest (the fallback for hosts that never send - lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and - `datasette serve` (cli.py) calls it too. The fast path below checks - both `_startup_invoked` and `_setup_db_done` - not just the former - - so that a bare `await ds.invoke_startup()` made by a caller ahead of - `_startup_sequence()` (which only sets `_startup_invoked`) can't - make this method skip the immutable-database table-count precompute. - """ - if self._startup_invoked and self._setup_db_done: - return - async with self._startup_lock: - if self._startup_invoked and self._setup_db_done: - return - if not self._setup_db_done: - # First time server starts up, calculate table counts for - # immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - self._setup_db_done = True - await self.invoke_startup() - - def add_background_task(self, func, name=None) -> BackgroundTask: - """Register a piece of supervised background work, typically from - a plugin's ``startup`` hook. - - ``func`` must be a coroutine function taking one positional - argument, the ``Datasette`` instance - core calls ``func(self)``. - Callable any time after ``__init__``: if background tasks haven't - launched yet (the common case - most callers are ``startup`` hooks, - which run before launch), this buffers the registration until they - do; if they've already launched (e.g. called from a request - handler after the server is up), the task starts immediately. - - Returns a :class:`~datasette.background_tasks.BackgroundTask` - handle (``.name``, ``.state``, ``.task``, ``.exception``, - ``.started_at``, ``.function``, ``.cancel()``). - - ``name`` defaults to ``func.__qualname__``; on a name collision a - ``-2``, ``-3``, ... suffix is appended, since names are how - ``/-/tasks`` and log messages identify work. - """ - return self._background_tasks.add(func, name=name) - - async def start_background_tasks(self): - """Run startup (if it hasn't run yet) and launch every registered - background task. - - Public entry point for tests, embedders, and headless CLIs (the - ``datasette-rss``-style ``fetch --due`` shape) that want supervised - background tasks without running a server - equivalent to what - happens automatically via ASGI lifespan / the first-request - fallback in a served deployment. - """ - await self.invoke_startup() - await self._background_tasks.launch_all() - - async def _launch_background_tasks(self): - """Idempotently launch every registered background task. Private: - this is the entry point wired into the lifecycle trigger lists - (the second entry in both ``AsgiLifespan`` and - ``AsgiRunOnFirstRequest``'s ``on_startup``, after - ``_startup_sequence``) - not something plugins or embedders should - call directly; use ``add_background_task`` / - ``start_background_tasks`` instead. - - Positioned after ``_startup_sequence`` in both trigger lists so - launch always happens once every plugin's ``startup`` hook has had - a chance to register work - the ordering guarantee that makes - ``add_background_task`` useful. No-ops when - ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its - one-shot TestClient request flows through the full ASGI stack, - including the first-request fallback, but must never launch - long-lived background work). - """ - if self._suppress_background_tasks: - return - await self._background_tasks.launch_all() - - async def invoke_shutdown(self): - """Run the graceful teardown sequence: plugin ``shutdown`` hooks, - then cancel and drain supervised background tasks, then close - every database. - """ - if self._shutdown_invoked: - return - self._shutdown_invoked = True - for hook in pm.hook.shutdown(datasette=self): - try: - await await_me_maybe(hook) - except Exception: - logging.getLogger("datasette").exception("shutdown hook failed") - await self._background_tasks.cancel_all(grace=5.0) - self.close() - def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() + async def setup_db(): + # First time server starts up, calculate table counts for immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + + async def _close_on_shutdown(): + self.close() + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan( - asgi, - on_startup=[self._startup_sequence, self._launch_background_tasks], - on_shutdown=[self.invoke_shutdown], - ) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) - asgi = AsgiRunOnFirstRequest( - asgi, - on_startup=[self._startup_sequence, self._launch_background_tasks], - ) - # Outermost, so spans from plugin middleware and first-request - # startup are children of the request span - asgi = TelemetryMiddleware(asgi) return asgi @@ -3207,50 +2860,6 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) - match, view = resolve_routes(self.routes, path) - is_static = view is favicon or getattr(view, "_datasette_static", False) - original_send = send - - async def send(message): - if message["type"] == "http.response.start" and not ( - is_static and message["status"] in (200, 304) - ): - # Decide privacy after rendering, including for streaming responses - # and error handlers. A public primary resource can still include - # private labels, actor navigation, or cookie-dependent content. - headers = list(message.get("headers", [])) - personalized = ( - request.actor is not None - or "cookie" in request.headers - or "authorization" in request.headers - or any(key.lower() == b"set-cookie" for key, _ in headers) - ) - if personalized: - headers = [ - (key, value) - for key, value in headers - if key.lower() != b"cache-control" - ] - headers.append((b"cache-control", b"private, no-store")) - - # Anonymous responses must not be reused for credentialed requests. - # Preserve any additional variation specified by views or plugins. - vary = [ - part.strip() - for key, value in headers - if key.lower() == b"vary" - for part in value.split(b",") - if part.strip() - ] - if b"*" not in vary: - for name in (b"Cookie", b"Authorization"): - if name.lower() not in {part.lower() for part in vary}: - vary.append(name) - headers = [(k, v) for k, v in headers if k.lower() != b"vary"] - headers.append((b"vary", b", ".join(vary))) - message = dict(message, headers=headers) - await original_send(message) - # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -3290,18 +2899,12 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - request.scope = scope + + match, view = resolve_routes(self.routes, path) if match is None: return await self.handle_404(request, send) - # Now the route is known, add it to the request span - span = request_span(scope) - if span is not None: - route = match.re.pattern - span.set_attribute(HTTP_ROUTE, route) - span.update_name(f"{clamp_http_method(request.method)} {route}") - new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) request.scope = new_scope try: @@ -3612,14 +3215,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3666,10 +3269,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx2 + **kwargs: Additional arguments to pass to httpx Returns: - httpx2.Response: The response from the request + httpx.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3678,16 +3281,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/background_tasks.py b/datasette/background_tasks.py deleted file mode 100644 index 6b34fd85..00000000 --- a/datasette/background_tasks.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Supervised background-task registration for Datasette core. - -Plugins that need long-lived background work (a polling loop, a queue -consumer, a scheduled job runner) register it with -``datasette.add_background_task(func, name=None)`` - typically from a -``startup`` plugin hook - instead of fire-and-forgetting their own -``asyncio.create_task()``. Core owns: - -- **references**: every launched ``asyncio.Task`` is kept alive on a - :class:`BackgroundTaskSupervisor`, so it can never be silently garbage - collected the way an unreferenced ``create_task()`` call can be; -- **launch timing**: registered work is buffered until - :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to - happen only after *every* plugin's ``startup`` hook has finished - so - a task that depends on another plugin having registered something first - doesn't need ``tryfirst=True`` ordering tricks; -- **crash surfacing**: an unhandled exception in a background task is - logged with its full traceback to the ``datasette.background_tasks`` - logger and recorded on the handle, instead of becoming an "Task - exception was never retrieved" warning nobody sees; -- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels - every task still running and waits (with a grace period) for them to - actually stop. -""" - -from __future__ import annotations - -import asyncio -import datetime -import functools -import logging -from collections.abc import Awaitable, Callable - -logger = logging.getLogger("datasette.background_tasks") - - -def _utcnow_iso() -> str: - return datetime.datetime.now(datetime.timezone.utc).isoformat() - - -def _function_path(func: Callable) -> str: - """Describe the callable without guessing which plugin registered it.""" - while isinstance(func, functools.partial): - func = func.func - if not hasattr(func, "__qualname__"): - func = type(func).__call__ - return f"{func.__module__}.{func.__qualname__}" - - -class BackgroundTask: - """A handle to a single piece of supervised background work. - - States: ``registered`` (added but not yet launched) -> ``running`` -> - one of ``completed`` (returned cleanly), ``crashed`` (raised an - exception other than ``CancelledError`` - see ``.exception``), or - ``cancelled`` (``.cancel()`` was called, or it was still running at - shutdown). - """ - - def __init__( - self, - name: str, - func: Callable[[object], Awaitable[None]], - ): - self.name = name - self.state = "registered" - self.task: asyncio.Task | None = None - self.exception: BaseException | None = None - self.started_at: str | None = None - self.function = _function_path(func) - self._func = func - self._supervisor: BackgroundTaskSupervisor | None = None - - def cancel(self) -> None: - """Cancel this task. - - If it has already been launched, cancels the underlying - ``asyncio.Task`` - its state becomes ``cancelled`` once the - cancellation is observed (asynchronously, via the task's done - callback). If it has not been launched yet, this is a no-op as - far as asyncio is concerned (there's no task to cancel) but it - deregisters the handle from its supervisor so it never runs. - """ - if self.task is not None: - self.task.cancel() - elif self._supervisor is not None: - self._supervisor._deregister(self) - - def __repr__(self) -> str: - return f"" - - -class BackgroundTaskSupervisor: - """Owns registration and launch of every :class:`BackgroundTask` for a - single ``Datasette`` instance. - - Registration (:meth:`add`) is separate from launch - (:meth:`launch_all`): plugins register work whenever convenient - (typically from a ``startup`` hook, but request handlers can register - dynamic per-job work too), and it either sits buffered until - :meth:`launch_all` runs, or - if :meth:`launch_all` has already run - - starts immediately. - - Strong references to every :class:`BackgroundTask` (and its - ``asyncio.Task``) are kept for the life of the instance, by design - - that's what makes the enrichments-style "fire-and-forget task gets - garbage collected mid-flight" bug impossible here. There is currently - no pruning of completed/crashed/cancelled tasks, so a plugin that - dynamically registers many short-lived tasks over a long process - lifetime (a per-job registration pattern, e.g. one task per queued - job) will grow this list without bound. That's an accepted v1 - trade-off in favour of full introspection (``/-/tasks``); revisit - with a pruning or capping policy if unbounded growth is reported in - practice. - """ - - def __init__(self, datasette): - self._datasette = datasette - self._tasks: list[BackgroundTask] = [] - self._names = set() - self._launched = False - self._lock = asyncio.Lock() - - def add(self, func, name=None) -> BackgroundTask: - base_name = name or getattr(func, "__qualname__", None) or repr(func) - actual_name = self._unique_name(base_name) - handle = BackgroundTask(actual_name, func) - handle._supervisor = self - self._tasks.append(handle) - self._names.add(actual_name) - if self._launched: - self._launch_one(handle) - return handle - - def _unique_name(self, base_name: str) -> str: - if base_name not in self._names: - return base_name - n = 2 - while f"{base_name}-{n}" in self._names: - n += 1 - return f"{base_name}-{n}" - - def _deregister(self, handle: BackgroundTask) -> None: - try: - self._tasks.remove(handle) - except ValueError: - pass - self._names.discard(handle.name) - - def _launch_one(self, handle: BackgroundTask) -> None: - handle.state = "running" - handle.started_at = _utcnow_iso() - handle.task = asyncio.create_task( - handle._func(self._datasette), name=handle.name - ) - handle.task.add_done_callback(functools.partial(_on_task_done, handle)) - - async def launch_all(self) -> None: - """Launch every currently-registered task that hasn't launched - yet. Idempotent and safe to call concurrently: subsequent (or - racing) calls are no-ops once the first has set ``self._launched``. - """ - if self._launched: - return - async with self._lock: - if self._launched: - return - self._launched = True - for handle in list(self._tasks): - if handle.task is None: - self._launch_one(handle) - - async def cancel_all(self, grace: float = 5.0) -> None: - """Cancel every task that isn't already done, then wait up to - ``grace`` seconds for them to actually finish. Stragglers still - running after that are logged by name (but left to finish or not - on their own - this does not forcibly kill them, asyncio has no - mechanism for that). - """ - handles_by_task = { - handle.task: handle for handle in self._tasks if handle.task is not None - } - pending = [task for task in handles_by_task if not task.done()] - for task in pending: - task.cancel() - if not pending: - return - _done, not_done = await asyncio.wait(pending, timeout=grace) - if not_done: - names = sorted(handles_by_task[task].name for task in not_done) - logger.warning( - "%d background task(s) did not finish within the %.1fs grace " - "period after cancellation: %s", - len(names), - grace, - ", ".join(names), - ) - - def tasks(self) -> list[BackgroundTask]: - """Return every registered :class:`BackgroundTask`, launched or - not, in registration order. Used by the ``/-/tasks`` debug - endpoint. - """ - return list(self._tasks) - - @property - def launched(self) -> bool: - """Whether :meth:`launch_all` has run yet - lets ``/-/tasks`` - distinguish "no tasks registered" from "tasks registered but - nothing has armed the launch yet" without reaching for the - private ``_launched`` attribute. - """ - return self._launched - - -def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None: - if task.cancelled(): - handle.state = "cancelled" - return - exc = task.exception() - if exc is not None: - handle.state = "crashed" - handle.exception = exc - logger.error("Background task %r crashed", handle.name, exc_info=exc) - return - handle.state = "completed" diff --git a/datasette/cli.py b/datasette/cli.py index e83de93a..57db83b6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -157,11 +157,7 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - - def _inspect_tables(conn): - return inspect_tables(conn, {}) - - tables = await database.execute_fn(_inspect_tables) + tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) data[name] = { "hash": database.hash, "size": database.size, @@ -501,7 +497,6 @@ def uninstall(packages, yes): "--internal", type=click.Path(), help="Path to a persistent Datasette internal SQLite database", - envvar="DATASETTE_INTERNAL", ) def serve( files, @@ -668,6 +663,16 @@ def serve( # Private utility mechanism for writing unit tests return ds + # Run async soundness checks before startup hooks, since invoke_startup + # now populates internal tables which requires querying each database + run_sync(lambda: check_databases(ds)) + + # Run the "startup" plugin hooks + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -675,19 +680,6 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: - # --get means we don't run Uvicorn at all - run_sync(lambda: check_databases(ds)) - - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - - # --get never launches background tasks: TestClient's request below - # flows through the full ASGI stack, including the - # AsgiRunOnFirstRequest fallback, which would otherwise launch them. - ds._suppress_background_tasks = True - client = TestClient(ds) request_headers = {} if token: @@ -712,54 +704,34 @@ def serve( sys.exit(exit_code) return - # check_databases, invoke_startup() and the uvicorn server all run on a - # single event loop, so that anything a plugin's "startup" hook schedules - # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is - # still alive when the server starts handling requests. - async def _serve_async(): - # Populate internal catalog tables before invoke_startup - await check_databases(ds) - - # Run the full startup sequence (immutable-database table-count - # precompute + the "startup" plugin hooks) via the same entry point - # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when - # uvicorn's lifespan.startup fires moments later. - try: - await ds._startup_sequence() - except StartupError as e: - raise click.ClickException(e.args[0]) - - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = await initial_path_for_datasette(ds) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) - await server.serve() - - asyncio.run(_serve_async()) + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = run_sync(lambda: initial_path_for_datasette(ds)) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + uvicorn.run(ds.app(), **uvicorn_kwargs) @cli.command() diff --git a/datasette/database.py b/datasette/database.py index 542b3012..e162d34e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,54 +1,18 @@ import asyncio import atexit -import contextvars import inspect import os import queue import sys import tempfile import threading -import time import uuid from collections import namedtuple from pathlib import Path import sqlite_utils -from opentelemetry import context as otel_context_api -from opentelemetry.trace import Status, StatusCode from .inspect import inspect_hash -from .telemetry import ( - callback_name, - linked_root_span_kwargs, - record_operation_duration, - record_query_interrupted, - record_write_queue_wait, - sql_attribute, - sql_operation_name, - tracer, -) -from .telemetry_registry import ( - CALLBACK, - DB_NAMESPACE, - DB_OPERATION_NAME, - DB_QUERY, - DB_QUERY_EXECUTE, - DB_QUERY_TEXT, - DB_SYSTEM, - DB_WRITE_EXECUTE, - DB_WRITE_QUEUE_WAIT, - EXECUTEMANY, - EXECUTESCRIPT, - INTERRUPTED, - ISOLATED_CONNECTION, - PARAM_COUNT, - PARAM_SETS, - ROWS_RETURNED, - SQL_ERROR_SUPPRESSED, - TIME_LIMIT_MS, - TRANSACTION, - TRUNCATED, -) from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -65,7 +29,7 @@ from .utils import ( table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names +from .utils.sqlite import sqlite_hidden_table_names connections = threading.local() @@ -121,7 +85,6 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None - self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -130,9 +93,8 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - # Track file and memory connections, including reads on worker threads, - # so close() can release all of them from the calling thread. - self._all_connections = [] + # This is used to track all file connections so they can be closed + self._all_file_connections = [] if not is_temp_disk: self.mode = mode @@ -183,12 +145,9 @@ class Database: ) if not write: conn.execute("PRAGMA query_only=1") - self._all_connections.append(conn) return conn if self.is_memory: - conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False) - self._all_connections.append(conn) - return conn + return sqlite3.connect(":memory:", uri=True) # mode=ro or immutable=1? if self.is_mutable: @@ -205,7 +164,7 @@ class Database: conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) - self._all_connections.append(conn) + self._all_file_connections.append(conn) if self.is_temp_disk and not self._wal_enabled: conn.execute("PRAGMA journal_mode=WAL") self._wal_enabled = True @@ -242,13 +201,13 @@ class Database: except Exception: # noqa: BLE001, S110 # Shutdown teardown - a failed pending write must not block close() pass - # Close anything still tracked in _all_connections - for connection in self._all_connections: + # Close anything still tracked in _all_file_connections + for connection in self._all_file_connections: try: connection.close() except Exception: # noqa: BLE001, S110 pass - self._all_connections = [] + self._all_file_connections = [] # Drop per-thread cached read connections we can reach try: delattr(connections, self._thread_local_id) @@ -287,45 +246,21 @@ class Database: return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, transaction=True, - time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def execute_sql(conn): + def _inner(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - def _inner(conn): - try: - if time_limit_ms is None: - return execute_sql(conn) - with sqlite_timelimit(conn, time_limit_ms): - return execute_sql(conn) - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - raise - - with trace( # noqa: SIM117 - "sql", database=self.name, sql=sql.strip(), params=params - ): - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - operation_name = sql_operation_name(sql) - if operation_name: - span.set_attribute(DB_OPERATION_NAME, operation_name) - if params: - span.set_attribute(PARAM_COUNT, len(params)) - with record_operation_duration(self.name, "write"): - results = await self._execute_write_fn( - _inner, block=block, request=request, transaction=transaction - ) + with trace("sql", database=self.name, sql=sql.strip(), params=params): + results = await self.execute_write_fn( + _inner, block=block, request=request, transaction=transaction + ) return results async def execute_write_script(self, sql, block=True, request=None): @@ -334,19 +269,10 @@ class Database: def _inner(conn): return conn.executescript(sql) - with trace( # noqa: SIM117 - "sql", database=self.name, sql=sql.strip(), executescript=True - ): - # No db.operation.name, since the script can contain multiple statements - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - span.set_attribute(EXECUTESCRIPT, True) - with record_operation_duration(self.name, "write"): - results = await self._execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + with trace("sql", database=self.name, sql=sql.strip(), executescript=True): + results = await self.execute_write_fn( + _inner, block=block, transaction=False, request=request + ) return results async def execute_write_many(self, sql, params_seq, block=True, request=None): @@ -366,19 +292,9 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - span.set_attribute(EXECUTEMANY, True) - operation_name = sql_operation_name(sql) - if operation_name: - span.set_attribute(DB_OPERATION_NAME, operation_name) - with record_operation_duration(self.name, "write"): - results, count = await self._execute_write_fn( - _inner, block=block, request=request - ) - span.set_attribute(PARAM_SETS, count) + results, count = await self.execute_write_fn( + _inner, block=block, request=request + ) kwargs["count"] = count return results @@ -395,58 +311,31 @@ class Database: finally: isolated_connection.close() try: - self._all_connections.remove(isolated_connection) + self._all_file_connections.remove(isolated_connection) except ValueError: - # May already have been cleared by close(). + # Was probably a memory connection pass - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(CALLBACK, callback_name(fn)) - # Immutable databases run this on the read pool, not the write queue - with record_operation_duration(self.name, "write" if write else "read"): - if self.ds.executor is None: - # non-threaded mode - return _run() - if not write: - # Immutable database - no writes can ever occur, so there - # is no write queue to block; run against a fresh - # read-only connection - ctx = contextvars.copy_context() - return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, ctx.run, _run - ) - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there is no + # write queue to block; run against a fresh read-only connection + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() - def _analyze_sql(conn): - return analyze_sql_tables(conn, sql, params, database_name=self.name) - - return await self.execute_isolated_fn(_analyze_sql) + return await self.execute_isolated_fn( + lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) + ) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): - """Run `fn(conn)` on the write connection, traced as a `db.query` span. - - The SQL-string write methods call `_execute_write_fn()` directly to - avoid creating a second span. - """ - self._check_not_closed() - # Record the name before _wrap_fn_with_hooks() wraps fn - name = callback_name(fn) - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(CALLBACK, name) - with record_operation_duration(self.name, "write"): - return await self._execute_write_fn( - fn, block=block, transaction=transaction, request=request - ) - - async def _execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() pending_events = [] @@ -465,15 +354,6 @@ class Database: result = fn(self._write_connection) else: result = fn(self._write_connection) - if not block: - # There is no write thread here, so the write has already - # finished. Hand back the same (task_id, reply_future) shape - # _send_to_write_thread() returns, with the future already - # resolved, so the block=False path below is identical in - # both modes. - reply_future = asyncio.get_running_loop().create_future() - reply_future.set_result(result) - result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -545,22 +425,11 @@ class Database: ) self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid4() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() reply_future = loop.create_future() - # Capture the OpenTelemetry context and enqueue time for the write thread self._write_queue.put( - WriteTask( - fn, - task_id, - loop, - reply_future, - isolated_connection, - transaction, - otel_context_api.get_current(), - time.time_ns(), - block, - ) + WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) ) if block: return await reply_future @@ -574,8 +443,6 @@ class Database: conn = None try: conn = self.connect(write=True) - # Threads do not inherit the caller's context, so any spans - # created by prepare_connection hooks here are root spans self.ds._prepare_connection(conn, self.name) except Exception as e: # noqa: BLE001 # Stored and re-raised to whoever queues the next write @@ -590,101 +457,42 @@ class Database: # Best-effort close as the write thread exits pass return - # block=True: the caller awaits the result, so the write spans - # are children of the caller's span. The token must be detached - # in the finally block or the context leaks into later writes. - # block=False: the caller may finish first, so the write spans - # are root spans with a link back to the caller's span. - token = None - write_span_kwargs = {} - if task.block: - token = otel_context_api.attach(task.otel_context) + exception = None + result = None + if conn_exception is not None: + exception = conn_exception + elif task.isolated_connection: + try: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + except Exception as e: # noqa: BLE001 + # Write thread must survive any task failure or the database wedges + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e else: - write_span_kwargs = linked_root_span_kwargs(task.otel_context) - try: - exception = None - result = None - # Span covers the time from enqueue to dequeue - dequeued_at_ns = time.time_ns() - tracer.start_span( - DB_WRITE_QUEUE_WAIT, - start_time=task.enqueued_at_ns, - **write_span_kwargs, - ).end(end_time=dequeued_at_ns) - record_write_queue_wait(self.name, dequeued_at_ns - task.enqueued_at_ns) - if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - try: - with tracer.start_as_current_span( - DB_WRITE_EXECUTE, **write_span_kwargs - ) as span: - span.set_attribute( - ISOLATED_CONNECTION, - task.isolated_connection, - ) - span.set_attribute(TRANSACTION, task.transaction) - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - finally: - isolated_connection.close() - try: - self._all_connections.remove(isolated_connection) - except ValueError: - # May already have been cleared by close(). - pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - else: - try: - with tracer.start_as_current_span( - DB_WRITE_EXECUTE, **write_span_kwargs - ) as span: - span.set_attribute( - ISOLATED_CONNECTION, - task.isolated_connection, - ) - span.set_attribute(TRANSACTION, task.transaction) - if task.transaction: - with conn: - conn.execute("BEGIN IMMEDIATE") - result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) - finally: - if token is not None: - otel_context_api.detach(token) + try: + if task.transaction: + with conn: + conn.execute("BEGIN IMMEDIATE") + result = task.fn(conn) + else: + result = task.fn(conn) + except Exception as e: # noqa: BLE001 + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) async def execute_fn(self, fn): - """Run `fn(conn)` on a read connection, traced as a `db.query` span. - - `execute()` calls `_execute_fn()` directly to avoid creating a second - span. - """ - self._check_not_closed() - - def fn_in_execute_span(conn): - # Runs on the worker thread - with tracer.start_as_current_span(DB_QUERY_EXECUTE): - return fn(conn) - - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(CALLBACK, callback_name(fn)) - with record_operation_duration(self.name, "read"): - return await self._execute_fn(fn_in_execute_span) - - async def _execute_fn(self, fn): self._check_not_closed() if self.ds.executor is None: # non-threaded mode @@ -704,11 +512,7 @@ class Database: with self._pending_execute_futures_lock: self._check_not_closed() - # Run in a copy of the caller's context so spans created in the - # thread have the correct parent. This needs a fresh copy for - # each submit, since a Context cannot be entered concurrently. - ctx = contextvars.copy_context() - future = self.ds.executor.submit(ctx.run, in_thread) + future = self.ds.executor.submit(in_thread) self._pending_execute_futures.add(future) future.add_done_callback(self._remove_pending_execute_future) return await asyncio.wrap_future(future) @@ -725,101 +529,44 @@ class Database: """Executes sql against db_name in a thread""" self._check_not_closed() page_size = page_size or self.ds.page_size - time_limit_ms = self.ds.sql_time_limit_ms - # Callers that pass a shorter custom_time_limit, such as table counts - # and facet suggestions, expect timeouts, so they are not span errors - timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms - if timeout_expected: - time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): - # Expected timeouts and errors with log_sql_errors=False are not - # recorded as span errors, so exceptions are handled explicitly - with tracer.start_as_current_span( - DB_QUERY_EXECUTE, - record_exception=False, - set_status_on_exception=False, - ) as execute_span: + time_limit_ms = self.ds.sql_time_limit_ms + if custom_time_limit and custom_time_limit < time_limit_ms: + time_limit_ms = custom_time_limit + + with sqlite_timelimit(conn, time_limit_ms): try: - with sqlite_timelimit(conn, time_limit_ms): - try: - cursor = conn.cursor() - cursor.execute(sql, params if params is not None else {}) - max_returned_rows = self.ds.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - if log_sql_errors: - sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" - ) - sys.stderr.flush() - raise - except QueryInterrupted as e: - if not timeout_expected: - execute_span.record_exception(e) - execute_span.set_status(Status(StatusCode.ERROR, str(e))) - raise - except Exception as e: - if log_sql_errors: - execute_span.record_exception(e) - execute_span.set_status(Status(StatusCode.ERROR, str(e))) - raise - - if truncate: - return Results(rows, truncated, cursor.description) - - else: - return Results(rows, False, cursor.description) - - with trace( # noqa: SIM117 - "sql", database=self.name, sql=sql.strip(), params=params - ): - with tracer.start_as_current_span( - DB_QUERY, - kind=DB_QUERY.kind, - record_exception=False, - set_status_on_exception=False, - ) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - span.set_attribute(TIME_LIMIT_MS, time_limit_ms) - operation_name = sql_operation_name(sql) - if operation_name: - span.set_attribute(DB_OPERATION_NAME, operation_name) - if params: - span.set_attribute(PARAM_COUNT, len(params)) - try: - with record_operation_duration(self.name, "read"): - results = await self._execute_fn(sql_operation_in_thread) - except QueryInterrupted as e: - span.set_attribute(INTERRUPTED, True) - if not timeout_expected: - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - record_query_interrupted(self.name) - raise - except Exception as e: - # log_sql_errors=False callers, such as facet suggestion, - # expect some queries to fail - if log_sql_errors: - span.record_exception(e) - span.set_status(Status(StatusCode.ERROR, str(e))) + cursor = conn.cursor() + cursor.execute(sql, params if params is not None else {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] else: - span.set_attribute(SQL_ERROR_SUPPRESSED, True) + rows = cursor.fetchall() + truncated = False + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + sys.stderr.write( + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + ) + sys.stderr.flush() raise - span.set_attribute(TRUNCATED, results.truncated) - span.set_attribute(ROWS_RETURNED, len(results.rows)) + + if truncate: + return Results(rows, truncated, cursor.description) + + else: + return Results(rows, False, cursor.description) + + with trace("sql", database=self.name, sql=sql.strip(), params=params): + results = await self.execute_fn(sql_operation_in_thread) return results @property @@ -910,32 +657,17 @@ class Database: ) return [r[0] for r in results.rows] - # Named functions rather than lambdas give more useful datasette.callback - # span attributes - async def table_columns(self, table): - def _table_columns(conn): - return table_columns(conn, table) - - return await self.execute_fn(_table_columns) + return await self.execute_fn(lambda conn: table_columns(conn, table)) async def table_column_details(self, table): - def _table_column_details(conn): - return table_column_details(conn, table) - - return await self.execute_fn(_table_column_details) + return await self.execute_fn(lambda conn: table_column_details(conn, table)) async def primary_keys(self, table): - def _primary_keys(conn): - return detect_primary_keys(conn, table) - - return await self.execute_fn(_primary_keys) + return await self.execute_fn(lambda conn: detect_primary_keys(conn, table)) async def fts_table(self, table): - def _fts_table(conn): - return detect_fts(conn, table) - - return await self.execute_fn(_fts_table) + return await self.execute_fn(lambda conn: detect_fts(conn, table)) async def label_column_for_table(self, table): explicit_label_column = (await self.ds.table_config(self.name, table)).get( @@ -1027,17 +759,6 @@ class Database: return hidden_tables - async def derived_table_dependencies(self): - """Return implementation tables and the tables they derive from.""" - schema_version = (await self.execute("PRAGMA schema_version")).first()[0] - if ( - self._cached_derived_table_dependencies is None - or self._cached_derived_table_dependencies[0] != schema_version - ): - dependencies = await self.execute_fn(sqlite_derived_table_dependencies) - self._cached_derived_table_dependencies = (schema_version, dependencies) - return self._cached_derived_table_dependencies[1] - async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] @@ -1133,28 +854,16 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( - "block", - "enqueued_at_ns", "fn", "isolated_connection", "loop", - "otel_context", "reply_future", "task_id", "transaction", ) def __init__( - self, - fn, - task_id, - loop, - reply_future, - isolated_connection, - transaction, - otel_context, - enqueued_at_ns, - block, + self, fn, task_id, loop, reply_future, isolated_connection, transaction ): self.fn = fn self.task_id = task_id @@ -1162,9 +871,6 @@ class WriteTask: self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction - self.otel_context = otel_context - self.enqueued_at_ns = enqueued_at_ns - self.block = block def _deliver_write_result(task, result, exception): diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index 6def3698..f90a733e 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,17 +6,6 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType -_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) - - -def _normalize_http_url(value): - if not isinstance(value, str): - return None - normalized = value.strip() - if not _HTTP_URL_RE.fullmatch(normalized): - return None - return normalized - class UrlColumnType(ColumnType): name = "url" @@ -26,10 +15,7 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - normalized = _normalize_http_url(value) - if normalized is None: - return markupsafe.escape(value.strip()) - escaped = markupsafe.escape(normalized) + escaped = markupsafe.escape(value.strip()) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -37,7 +23,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if _normalize_http_url(value) is None: + if not re.match(r"^https?://\S+$", value.strip()): return "Invalid URL" return None diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index a4f5a4de..4494f07f 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -92,13 +92,6 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - # Resolve identity keys once per action, rather than scanning the - # restriction allowlist for every configured table's allow block. - self.restricted_table_keys = { - (db, self.action_obj.normalize_child(table) if self.action_obj else table) - for db, table in self.restricted_tables - } - def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: @@ -132,10 +125,8 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - child_key = ( - self.action_obj.normalize_child(child) if self.action_obj else child - ) - if (parent, child_key) in self.restricted_table_keys: + table_actions = table_restrictions.get(child, []) + if self.action_checks.intersection(table_actions): return True else: # Parent query should proceed if any child in this database is allowlisted diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index d30ebd3f..88e1d274 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -185,15 +185,11 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - action_obj = datasette.actions.get(action) - normalize = action_obj.normalize_child if action_obj else lambda name: name - for table_name, table_allowed in ( - restrictions.get("r", {}).get(database, {}).items() - ): - if normalize(table_name) == normalize(table): - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py deleted file mode 100644 index 11fd4008..00000000 --- a/datasette/default_permissions/sqlite_statistics.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Default table-access policy for SQLite optimizer statistics.""" - -import json - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - - -@hookimpl -def permission_resources_sql(action): - if action != "view-table": - return None - return PermissionSQL( - sql=""" - SELECT database_name AS parent, value AS child, 0 AS allow, - 'SQLite statistics tables are denied by default' AS reason - FROM catalog_databases - CROSS JOIN json_each(:sqlite_statistics_names) - """, - params={ - "sqlite_statistics_names": json.dumps( - ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] - ) - }, - ) diff --git a/datasette/facets.py b/datasette/facets.py index 69ac2c42..8c09e1dc 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -39,7 +39,7 @@ def load_facet_configs(request, table_config): ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): - if key == "_facet" or key.startswith("_facet_"): + if key.startswith("_facet"): # Figure out the facet type if key == "_facet": type = "column" @@ -264,15 +264,10 @@ class ColumnFacet(Facet): column_qs = column if column.startswith("_"): column_qs = f"{column}__exact" - selected_args = { - key: str(row["value"]) - for key in (column_qs, f"{column}__exact") - if (key, str(row["value"])) in qs_pairs - } - selected = bool(selected_args) + selected = (column_qs, str(row["value"])) in qs_pairs if selected: toggle_path = path_with_removed_args( - self.request, selected_args + self.request, {column_qs: str(row["value"])} ) else: toggle_path = path_with_added_args( diff --git a/datasette/filters.py b/datasette/filters.py index 0499d086..3cfb36e5 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,9 +1,8 @@ import json -import math from typing import ClassVar from datasette import hookimpl -from datasette.resources import DatabaseResource, TableResource +from datasette.resources import DatabaseResource from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError @@ -52,20 +51,13 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which trusted fts_table to use. Query string parameters can - # repeat this mapping (for backwards compatibility), but must not select - # a different table or primary key. + # Figure out which fts_table to use table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = table_metadata.get("fts_table") + fts_table = request.args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = table_metadata.get("fts_pk", "rowid") - requested_fts_table = request.args.get("_fts_table") - requested_fts_pk = request.args.get("_fts_pk") - if (requested_fts_table and requested_fts_table != fts_table) or ( - requested_fts_pk and requested_fts_pk != fts_pk - ): - raise BadRequest("Invalid _fts_table or _fts_pk") + fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) search_args = { key: request.args[key] for key in request.args @@ -83,11 +75,6 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=fts_table), - actor=request.actor, - ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -148,11 +135,6 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=through_table), - actor=request.actor, - ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) fk_to_us = next( @@ -203,17 +185,6 @@ class Filter: raise NotImplementedError -def _coerce_numeric_filter_value(value): - try: - return int(value) - except ValueError: - try: - converted = float(value) - except ValueError: - return value - return converted if math.isfinite(converted) else value - - class TemplatedFilter(Filter): def __init__( self, @@ -235,8 +206,8 @@ class TemplatedFilter(Filter): def where_clause(self, table, column, value, param_counter): converted = self.format.format(value) - if self.numeric: - converted = _coerce_numeric_filter_value(converted) + if self.numeric and converted.isdigit(): + converted = int(converted) if self.no_argument: kwargs = {"c": _quote_sqlite_identifier(column)} converted = None diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index ef6c7b7e..c36d5dbe 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -59,10 +59,6 @@ def handle_exception(datasette, request, exception): body = dict(info) body.update(error_body(plain_message or message, status)) return Response.json(body, status=status, headers=headers) - if request.path.split("?")[0].endswith(".csv"): - return Response.text( - plain_message or message, status=status, headers=headers - ) info.update( { "ok": False, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 49d8e8ea..f89f2f36 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -9,11 +9,6 @@ def startup(datasette): """Fires directly after Datasette first starts running""" -@hookspec -def shutdown(datasette): - """Called once when the Datasette server is shutting down""" - - @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" @@ -50,7 +45,7 @@ def extra_body_script( def extra_template_vars( template, database, table, columns, view_name, request, datasette ): - """Extra template variables to be made available to the template - can return dict, None, callable or awaitable""" + """Extra template variables to be made available to the template - can return dict or callable or awaitable""" @hookspec diff --git a/datasette/permissions.py b/datasette/permissions.py index 2d242560..e03b065c 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -3,10 +3,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple -_SQLITE_IDENTIFIER_CASE = str.maketrans( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" -) - # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False @@ -53,15 +49,6 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables - case_insensitive_child: bool = False - - @classmethod - def normalize_child(cls, child: str | None) -> str | None: - """Return a comparison key without changing the resource's display name.""" - if cls.case_insensitive_child and child is not None: - # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. - return child.translate(_SQLITE_IDENTIFIER_CASE) - return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -159,11 +146,6 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed - def normalize_child(self, child: str | None) -> str | None: - if self.resource_class is None: - return child - return self.resource_class.normalize_child(child) - @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 6a4d7da7..9cf94079 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -18,7 +18,6 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", - "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", diff --git a/datasette/resources.py b/datasette/resources.py index 29bf7b1e..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,7 +25,6 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource - case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/app.css b/datasette/static/app.css index 0297371c..d101e4b7 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1,144 +1,3 @@ -/* Shared modal styles. */ -datasette-modal { - display: contents; -} - -dialog.datasette-modal { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; - max-height: calc(100dvh - 32px); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.datasette-modal[open] { - display: flex; - flex-direction: column; -} - -dialog.datasette-modal::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -@keyframes datasette-modal-slide-in { - from { opacity: 0; transform: translateY(-20px) scale(0.95); } - to { opacity: 1; transform: translateY(0) scale(1); } -} - -@keyframes datasette-modal-fade-in { - from { opacity: 0; } - to { opacity: 1; } -} - -:where(.datasette-modal) .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; -} - -:where(.datasette-modal) .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -:where(.datasette-modal) .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -:where(.datasette-modal) .modal-body { - min-height: 0; - overflow: auto; - padding: 16px 24px 24px; -} - -:where(.datasette-modal) .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -:where(.datasette-modal) .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -:where(.datasette-modal) .modal-btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -:where(.datasette-modal) .modal-btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -:where(.datasette-modal) .modal-btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -:where(.datasette-modal) .modal-btn-primary { - background: var(--accent); - color: #fff; -} - -:where(.datasette-modal) .modal-btn-primary:hover { - background: #1949b8; -} - -:where(.datasette-modal) .modal-btn:disabled { - opacity: 0.65; - cursor: wait; -} - -@media (prefers-reduced-motion: reduce) { - dialog.datasette-modal, - dialog.datasette-modal::backdrop { - animation: none; - } -} - /* Reset and Page Setup ==================================================== */ /* Reset from http://meyerweb.com/eric/tools/css/reset/ @@ -204,7 +63,7 @@ em { } /* end reset */ -/* Shared modal CSS variables */ +/* Modal CSS variables (shared by web components via Shadow DOM) */ :root { --modal-backdrop-bg: rgba(0, 0, 0, 0.5); --modal-backdrop-blur: blur(4px); @@ -357,49 +216,6 @@ a:active { text-decoration: underline; } -.table-summary .count-all ~ .table-summary-description { - margin-left: 0.5rem; -} - -.table-summary .count-error:not(:empty) { - display: block; - margin-top: 0.25rem; - font-size: 0.875rem; - font-weight: 400; - line-height: 1.5; -} - -button.count-all { - background: none; - border: none; - padding: 3px 0; - margin-left: 0.25rem; - color: #276890; - font-family: inherit; - font-size: 0.8125rem; - font-weight: 400; - line-height: 1.5; - cursor: pointer; -} - -button.count-all:hover, -button.count-all:focus-visible { - text-decoration: underline; -} - -button.count-all:disabled { - color: #596478; - cursor: wait; -} - -@media (pointer: coarse) { - button.count-all { - min-height: 44px; - padding-left: 7px; - padding-right: 7px; - } -} - button.button-as-link { background: none; border: none; @@ -1122,552 +938,84 @@ p.zero-results { display: none; } -/* navigation-search */ -navigation-search { - display: contents; -} - -navigation-search dialog.datasette-modal { - max-width: 90vw; - width: 600px; - max-height: 80vh; -} - -navigation-search .search-container { - display: flex; - flex-direction: column; -} - -navigation-search .search-input-wrapper { - padding: 1.25rem; - border-bottom: 1px solid #e5e7eb; - display: flex; - gap: 0.5rem; - align-items: center; -} - -navigation-search .search-input { - width: 100%; - flex: 1; - min-width: 0; - padding: 0.75rem 1rem; - font-size: 1rem; - border: 2px solid #e5e7eb; - border-radius: 0.5rem; - outline: none; - transition: border-color 0.2s; - box-sizing: border-box; -} - -navigation-search .search-input:focus { - border-color: #2563eb; -} - -navigation-search .close-search { - background: transparent; - border: 1px solid transparent; - border-radius: 0.375rem; - color: #4b5563; - cursor: pointer; - flex: 0 0 auto; - font: inherit; - font-size: 1.5rem; - height: 2.75rem; - line-height: 1; - width: 2.75rem; -} - -navigation-search .close-search:hover, -navigation-search .close-search:focus { - background-color: #f3f4f6; - border-color: #d1d5db; -} - -navigation-search .results-container { - box-sizing: content-box; - height: calc(80vh - 180px); - padding: 0.5rem; -} - -navigation-search .results-list:empty { - display: none; -} - -navigation-search .result-item { - padding: 0.875rem 1rem; - cursor: pointer; - border-radius: 0.5rem; - transition: background-color 0.15s; - display: flex; - align-items: center; - gap: 0.75rem; -} - -navigation-search .result-item:hover { - background-color: #f3f4f6; -} - -navigation-search .result-item.selected { - background-color: #dbeafe; -} - -navigation-search .result-item > div { - flex: 1; - min-width: 0; -} - -navigation-search .jump-start-content { - border-bottom: 1px solid #e5e7eb; - margin-bottom: 0.5rem; - padding: 0.5rem 0.5rem 1rem; -} - -navigation-search .jump-start-content:empty { - display: none; -} - -navigation-search .result-name { - font-weight: 500; - color: #111827; -} - -navigation-search .result-label { - font-size: 0.875rem; - color: #4b5563; -} - -navigation-search .result-type { - color: #4b5563; - font-size: 0.75rem; - font-weight: 600; - text-transform: uppercase; -} - -navigation-search .result-url { - font-size: 0.875rem; - color: #6b7280; -} - -navigation-search .result-description { - color: #374151; - display: -webkit-box; - font-size: 0.8125rem; - line-height: 1.35; - margin-top: 0.35rem; - overflow: hidden; - -webkit-box-orient: vertical; - -webkit-line-clamp: 2; -} - -navigation-search .results-heading { - color: #4b5563; - font-size: 0.75rem; - font-weight: 600; - letter-spacing: 0; - padding: 0.5rem 1rem 0.25rem; - text-transform: uppercase; -} - -navigation-search .recent-actions { - padding: 0.25rem 1rem 0.75rem; -} - -navigation-search .clear-recent { - background: transparent; - border: 0; - color: #2563eb; - cursor: pointer; - font: inherit; - font-size: 0.875rem; - padding: 0; -} - -navigation-search .clear-recent:hover { - text-decoration: underline; -} - -navigation-search .no-results { - padding: 2rem; - text-align: center; - color: #6b7280; -} - -navigation-search .hint-text { - padding: 0.75rem 1.25rem; - font-size: 0.875rem; - color: #6b7280; - border-top: 1px solid #e5e7eb; - display: flex; - gap: 1rem; - flex-wrap: wrap; -} - -navigation-search .hint-text kbd { - background: #f3f4f6; - padding: 0.125rem 0.375rem; - border-radius: 0.25rem; - font-size: 0.75rem; - border: 1px solid #d1d5db; - font-family: monospace; -} - -navigation-search .visually-hidden { - border: 0; - clip: rect(0 0 0 0); - height: 1px; - margin: -1px; - overflow: hidden; - padding: 0; - position: absolute; - white-space: nowrap; - width: 1px; -} - -@media (max-width: 640px) { - navigation-search dialog.datasette-modal { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; +@keyframes datasette-modal-slide-in { + from { + opacity: 0; + transform: translateY(-20px) scale(0.95); } - - navigation-search .search-input-wrapper { - padding: 1rem; - } - - navigation-search .search-input { - font-size: 16px; - } - - navigation-search .result-item { - padding: 1rem 0.75rem; - } - - navigation-search .hint-text { - font-size: 0.8rem; - padding: 0.5rem 1rem; + to { + opacity: 1; + transform: translateY(0) scale(1); } } +@keyframes datasette-modal-fade-in { + from { opacity: 0; } + to { opacity: 1; } +} -/* column-chooser */ -column-chooser { - display: contents; +dialog.mobile-column-actions-dialog { --ink: #0f0f0f; --paper: #eef6ff; --muted: #6b6b6b; --rule: #d8e6f5; --accent: #1a56db; - --accent-light: #e8effd; --card: #ffffff; -} - -column-chooser * { - box-sizing: border-box; - margin: 0; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); padding: 0; -} - -column-chooser dialog.datasette-modal { - width: 100%; - max-width: 420px; - max-height: min(640px, calc(100vh - 32px)); - -webkit-user-select: none; - -webkit-touch-callout: none; - -webkit-tap-highlight-color: transparent; -} - -column-chooser dialog.datasette-modal[open] { - height: min(640px, calc(100vh - 32px)); -} - -column-chooser .modal-header { - padding: 20px 24px 16px; - justify-content: space-between; -} - -column-chooser .list-toolbar { - padding: 6px 24px; - border-bottom: 1px solid var(--rule); - display: flex; - gap: 12px; - flex-shrink: 0; -} - -column-chooser .list-toolbar button { - background: var(--accent-light); - border: 1px solid var(--rule); - border-radius: 4px; - font-family: inherit; - font-size: 0.75rem; - color: var(--accent); - cursor: pointer; - padding: 3px 10px; - transition: - background 0.12s, - color 0.12s; -} - -column-chooser .list-toolbar button:hover { - background: var(--accent); - color: white; -} - -column-chooser .list-wrap { - flex: 1; - padding: 0; - overflow-x: hidden; - position: relative; - overscroll-behavior: contain; - -webkit-overflow-scrolling: touch; -} - -column-chooser .list-wrap::before, -column-chooser .list-wrap::after { - content: ""; - position: sticky; - display: block; - left: 0; - right: 0; - height: 20px; - pointer-events: none; - z-index: 5; - transition: opacity 0.2s; -} - -column-chooser .list-wrap::before { - top: 0; - background: linear-gradient( - to bottom, - rgba(255, 255, 255, 0.9), - transparent - ); -} - -column-chooser .list-wrap::after { - bottom: 0; - background: linear-gradient(to top, rgba(255, 255, 255, 0.9), transparent); - margin-top: -20px; -} - -column-chooser .scroll-zone { - position: absolute; - left: 0; - right: 0; - height: 72px; - pointer-events: none; - z-index: 10; -} - -column-chooser .scroll-zone-top { - top: 0; -} - -column-chooser .scroll-zone-bot { - bottom: 0; -} - -column-chooser .drag-list { - list-style: none; - padding: 4px 0; -} - -column-chooser .drag-item { - display: flex; - align-items: center; - background: white; - border-bottom: 1px solid var(--rule); - user-select: none; - -webkit-user-select: none; - -webkit-touch-callout: none; - position: relative; - transition: background 0.08s; -} - -column-chooser .drag-item:last-child { - border-bottom: none; -} - -column-chooser .drag-handle { - display: flex; - align-items: center; - justify-content: center; - width: 48px; - height: 48px; - flex-shrink: 0; - cursor: grab; - color: #c8c4bc; - touch-action: none; - transition: color 0.15s; -} - -column-chooser .drag-handle:hover { - color: var(--accent); -} - -column-chooser .drag-handle svg { - pointer-events: none; - display: block; -} - -column-chooser .drag-item-content { - display: flex; - align-items: center; - flex: 1; - min-width: 0; - cursor: pointer; -} - -column-chooser .drag-item-check { - display: flex; - align-items: center; - width: 32px; - height: 48px; - flex-shrink: 0; -} - -column-chooser .drag-item-check input[type="checkbox"] { - width: 16px; - height: 16px; - accent-color: var(--accent); - cursor: pointer; -} - -column-chooser .drag-item-label { - flex: 1; - font-size: 0.9rem; - line-height: 48px; - padding-right: 16px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - cursor: default; -} - -column-chooser .drag-item.is-dragging { - opacity: 0; -} - -column-chooser .drop-indicator { - position: absolute; - left: 48px; - right: 0; - height: 2px; - background: var(--accent); - border-radius: 99px; - pointer-events: none; - z-index: 20; - display: none; -} - -column-chooser .drop-indicator.top { - top: -1px; - display: block; -} - -column-chooser .drop-indicator.bottom { - bottom: -1px; - display: block; -} - -column-chooser .drag-ghost { - position: fixed; - pointer-events: none; - z-index: 9999; - background: white; - border-radius: 6px; - box-shadow: - 0 8px 32px rgba(0, 0, 0, 0.18), - 0 2px 8px rgba(0, 0, 0, 0.1); - display: flex; - align-items: center; - border: 1.5px solid var(--accent-light); - opacity: 0.97; - will-change: transform; - font-family: - system-ui, - -apple-system, - sans-serif; -} - -column-chooser .scroll-pulse { - position: absolute; - left: 50%; - transform: translateX(-50%); - width: 32px; - height: 32px; - border-radius: 50%; - background: var(--accent); - opacity: 0; - pointer-events: none; - z-index: 10; - transition: opacity 0.15s; -} - -column-chooser .scroll-pulse.top { - top: 8px; -} - -column-chooser .scroll-pulse.bot { - bottom: 8px; -} - -column-chooser .scroll-pulse.active { - opacity: 0.18; - animation: column-chooser-pulse 0.8s ease-in-out infinite; -} - -@keyframes column-chooser-pulse { - 0%, - 100% { - transform: translateX(-50%) scale(1); - opacity: 0.18; - } - 50% { - transform: translateX(-50%) scale(1.5); - opacity: 0.07; - } -} - -column-chooser .modal-btn-primary { - color: white; -} - -column-chooser .modal-btn-primary:hover { - background: #1448c0; -} - -column-chooser .list-wrap::-webkit-scrollbar { - width: 5px; -} - -column-chooser .list-wrap::-webkit-scrollbar-track { - background: transparent; -} - -column-chooser .list-wrap::-webkit-scrollbar-thumb { - background: var(--rule); - border-radius: 99px; -} - -column-chooser input, -column-chooser textarea { - -webkit-user-select: auto; - user-select: auto; -} - -dialog.mobile-column-actions-dialog { + margin: auto; width: min(420px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(640px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.mobile-column-actions-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.mobile-column-actions-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .mobile-column-actions-dialog .modal-header { padding: 20px 24px 16px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.mobile-column-actions-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.mobile-column-actions-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; } .mobile-column-actions-dialog .list-wrap { flex: 1 1 auto; - padding: 0; + min-height: 0; + overflow-y: auto; overflow-x: hidden; position: relative; overscroll-behavior: contain; @@ -1794,12 +1142,102 @@ dialog.mobile-column-actions-dialog { font-size: 0.85em; } +.mobile-column-actions-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.mobile-column-actions-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.mobile-column-actions-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.mobile-column-actions-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.mobile-column-actions-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + dialog.set-column-type-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(520px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(720px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.set-column-type-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.set-column-type-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .set-column-type-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.set-column-type-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.set-column-type-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; } .set-column-type-status, @@ -1821,6 +1259,8 @@ dialog.set-column-type-dialog { } .set-column-type-options { + padding: 16px 24px 24px; + overflow-y: auto; display: grid; gap: 12px; } @@ -1862,6 +1302,60 @@ dialog.set-column-type-dialog { font-size: 0.9rem; } +.set-column-type-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.set-column-type-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.set-column-type-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.set-column-type-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.set-column-type-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.set-column-type-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.set-column-type-dialog .btn-primary:hover { + background: #1949b8; +} + +.set-column-type-dialog .btn:disabled { + opacity: 0.65; + cursor: wait; +} + .row-mutation-status { margin: 0 0 0.75rem; padding: 8px 10px; @@ -1895,11 +1389,46 @@ button.table-insert-row svg { } dialog.row-delete-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(440px, calc(100vw - 32px)); + max-width: 95vw; + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.row-delete-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.row-delete-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .row-delete-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; justify-content: flex-start; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .row-delete-dialog .modal-title { @@ -1908,6 +1437,9 @@ dialog.row-delete-dialog { gap: 0.35rem; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .row-delete-message, @@ -1939,12 +1471,94 @@ dialog.row-delete-dialog { .row-delete-dialog .modal-footer { padding: 18px 20px 14px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); margin-top: 18px; } +.row-delete-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.row-delete-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.row-delete-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.row-delete-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.row-delete-dialog .btn-primary:hover { + background: #1949b8; +} + +.row-delete-dialog .btn:disabled { + opacity: 0.65; + cursor: wait; +} + dialog.row-edit-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(720px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.row-edit-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.row-edit-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.row-edit-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .row-edit-dialog .modal-title { @@ -1953,6 +1567,9 @@ dialog.row-edit-dialog { gap: 0.35rem; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .row-edit-dialog .modal-title .row-dialog-action, @@ -2020,6 +1637,8 @@ dialog.row-edit-dialog { .row-edit-fields { display: grid; gap: 14px; + padding: 16px 24px 24px; + overflow-y: auto; } .row-edit-fields[hidden], @@ -2299,6 +1918,8 @@ textarea.row-edit-input { .row-edit-bulk { display: grid; gap: 8px; + padding: 16px 24px 24px; + overflow-y: auto; } .row-edit-bulk-editor { @@ -2318,7 +1939,7 @@ textarea.row-edit-input { justify-content: flex-start; } -.row-edit-bulk-actions .modal-btn { +.row-edit-bulk-actions .btn { padding-left: 12px; padding-right: 12px; } @@ -2562,6 +2183,17 @@ datasette-autocomplete input[type="text"], max-width: 46rem; } +.row-edit-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + .row-edit-mode-link { color: var(--accent); font-size: 0.9rem; @@ -2572,14 +2204,84 @@ datasette-autocomplete input[type="text"], display: none; } -.row-edit-dialog .modal-btn:disabled { +.row-edit-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.row-edit-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.row-edit-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.row-edit-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.row-edit-dialog .btn-primary:hover { + background: #1949b8; +} + +.row-edit-dialog .btn:disabled { opacity: 0.55; cursor: not-allowed; } dialog.table-create-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(980px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-create-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-create-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-create-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .table-create-dialog .modal-title { @@ -2587,6 +2289,9 @@ dialog.table-create-dialog { align-items: center; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .table-create-form { @@ -2614,6 +2319,8 @@ dialog.table-create-dialog { .table-create-fields { display: grid; gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; } .table-create-field { @@ -3023,6 +2730,17 @@ select.table-create-input { outline-offset: 1px; } +.table-create-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + .table-create-mode-link { color: var(--accent); font-size: 0.9rem; @@ -3033,7 +2751,39 @@ select.table-create-input { display: none; } -.table-create-dialog .modal-btn:disabled, +.table-create-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-create-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-create-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-create-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-create-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-create-dialog .btn:disabled, .table-create-add-column:disabled, .table-create-icon-button:disabled { opacity: 0.55; @@ -3041,8 +2791,46 @@ select.table-create-input { } dialog.table-alter-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(980px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-alter-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-alter-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-alter-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .table-alter-dialog .modal-title { @@ -3050,6 +2838,9 @@ dialog.table-alter-dialog { align-items: center; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .table-alter-form { @@ -3077,6 +2868,8 @@ dialog.table-alter-dialog { .table-alter-fields { display: grid; gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; } .table-alter-table-options { @@ -3110,6 +2903,8 @@ dialog.table-alter-dialog { .table-alter-review { display: grid; gap: 12px; + overflow-y: auto; + padding: 16px 24px 24px; } .table-alter-review[hidden] { @@ -3403,29 +3198,72 @@ select.table-alter-input { outline-offset: 1px; } -.table-alter-dialog .modal-btn-danger { +.table-alter-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.table-alter-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-alter-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-alter-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-alter-dialog .btn-danger { background: #b91c1c; color: #fff; margin-right: auto; } -.table-alter-dialog .modal-btn-danger:hover { +.table-alter-dialog .btn-danger:hover { background: #991b1b; } -.table-alter-dialog .modal-btn-danger:disabled, -.table-alter-dialog .modal-btn-danger:disabled:hover { +.table-alter-dialog .btn-danger:disabled, +.table-alter-dialog .btn-danger:disabled:hover { background: #d98c8c; color: #fff; } -.table-alter-dialog .modal-btn-primary:disabled, -.table-alter-dialog .modal-btn-primary:disabled:hover { +.table-alter-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-alter-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-alter-dialog .btn-primary:disabled, +.table-alter-dialog .btn-primary:disabled:hover { background: #a0aec0; color: #fff; } -.table-alter-dialog .modal-btn:disabled, +.table-alter-dialog .btn:disabled, .table-alter-add-column:disabled, .table-alter-icon-button:disabled { opacity: 0.55; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index 29729f27..198641f3 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -1,9 +1,7 @@ -let columnChooserInstanceCounter = 0; - class ColumnChooser extends HTMLElement { constructor() { super(); - this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`; + this.attachShadow({ mode: "open" }); // State this._items = []; @@ -28,60 +26,375 @@ class ColumnChooser extends HTMLElement { // Bound handlers this._onMove = this._onMove.bind(this); this._onUp = this._onUp.bind(this); - } - connectedCallback() { - if (this._modal) return; - this.innerHTML = ` - + this.shadowRoot.innerHTML = ` + + +
- Choose columns - + Choose columns +
- - + +
-
-
-
-
    +
    +
    +
    +
      - - - + + +
      -
      + `; // DOM refs - this._modal = this.querySelector("datasette-modal"); - this._listWrap = this.querySelector(".list-wrap"); - this._dragList = this.querySelector(".drag-list"); - this._pulseTop = this.querySelector(".scroll-pulse.top"); - this._pulseBot = this.querySelector(".scroll-pulse.bot"); - this._selectAllBtn = this.querySelector(".select-all"); - this._deselectAllBtn = this.querySelector(".deselect-all"); - this._cancelBtn = this.querySelector(".modal-btn-ghost"); - this._applyBtn = this.querySelector(".modal-btn-primary"); - this._countEl = this.querySelector(".modal-meta"); - this._footerEl = this.querySelector(".footer-info"); + this._dialog = this.shadowRoot.querySelector("dialog"); + this._listWrap = this.shadowRoot.getElementById("listWrap"); + this._dragList = this.shadowRoot.getElementById("dragList"); + this._pulseTop = this.shadowRoot.getElementById("pulseTop"); + this._pulseBot = this.shadowRoot.getElementById("pulseBot"); + this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); + this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); + this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); + this._applyBtn = this.shadowRoot.getElementById("applyBtn"); + this._countEl = this.shadowRoot.getElementById("selectedCount"); + this._footerEl = this.shadowRoot.getElementById("footerInfo"); // Event listeners this._selectAllBtn.addEventListener("click", () => this._selectAll()); this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); - this._cancelBtn.addEventListener("click", () => - this._modal.requestClose("cancel"), - ); + this._cancelBtn.addEventListener("click", () => this._close()); this._applyBtn.addEventListener("click", () => this._apply()); - this._modal.beforeClose = () => { - this._items = this._savedItems ? [...this._savedItems] : this._items; - this._checked = this._savedChecked - ? new Set(this._savedChecked) - : this._checked; - return true; - }; + this._dialog.addEventListener("click", (e) => { + if (e.target === this._dialog) this._close(); + }); + this._dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this._close(); + }); } /** @@ -101,11 +414,19 @@ class ColumnChooser extends HTMLElement { this._savedChecked = new Set(this._checked); this._render(); - this._modal.show(); + this._dialog.showModal(); } // ── Internal methods ── + _close() { + this._items = this._savedItems ? [...this._savedItems] : this._items; + this._checked = this._savedChecked + ? new Set(this._savedChecked) + : this._checked; + this._dialog.close(); + } + _selectAll() { this._items.forEach((col) => this._checked.add(col)); this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { @@ -124,7 +445,7 @@ class ColumnChooser extends HTMLElement { _apply() { const selected = this._items.filter((col) => this._checked.has(col)); - this._modal.close(); + this._dialog.close(); if (this._onApply) { this._onApply(selected); } @@ -151,13 +472,11 @@ class ColumnChooser extends HTMLElement { - + ${col}
      `; - li.querySelector(".drag-item-label").textContent = col; - li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); @@ -190,7 +509,7 @@ class ColumnChooser extends HTMLElement { this._ghostOffX = e.clientX - rect.left; this._ghostOffY = e.clientY - rect.top; - // Keep the drag preview inside the dialog so it stays above the backdrop. + // Build ghost inside shadow DOM this._ghost = document.createElement("div"); this._ghost.className = "drag-ghost"; this._ghost.style.width = rect.width + "px"; @@ -199,7 +518,7 @@ class ColumnChooser extends HTMLElement { this._ghost.querySelector(".drop-indicator")?.remove(); const h = this._ghost.querySelector(".drag-handle"); if (h) h.style.color = "var(--accent)"; - this._modal.dialog.appendChild(this._ghost); + this.shadowRoot.appendChild(this._ghost); srcEl.classList.add("is-dragging"); this._positionGhost(e.clientX, e.clientY); diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 0f61ebd6..9e8b93f6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -915,7 +915,6 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.modal.busy = isSaving; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { @@ -2044,7 +2043,8 @@ async function createTableFromDataPreview(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (tableUrl) { location.href = tableUrl; } else { @@ -2118,7 +2118,8 @@ async function saveTableCreateDialog(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (tableUrl) { location.href = tableUrl; } else { @@ -2140,6 +2141,18 @@ function confirmDiscardTableCreateChanges(state) { return window.confirm("Discard this new table?"); } +function closeTableCreateDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableCreateChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + function ensureTableCreateDialog(manager) { if (tableCreateDialogState) { return tableCreateDialogState; @@ -2148,8 +2161,7 @@ function ensureTableCreateDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = TABLE_CREATE_DIALOG_ID; dialog.className = "table-create-dialog"; dialog.setAttribute("aria-labelledby", "table-create-title"); @@ -2159,7 +2171,7 @@ function ensureTableCreateDialog(manager) {
      -
      +
      @@ -2186,15 +2198,14 @@ function ensureTableCreateDialog(manager) {
      Create table from data - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); tableCreateDialogState = { - modal: modal, dialog: dialog, form: dialog.querySelector(".table-create-form"), title: dialog.querySelector(".modal-title"), @@ -2214,6 +2225,8 @@ function ensureTableCreateDialog(manager) { manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), + currentButton: null, + shouldRestoreFocus: true, isSaving: false, mode: "manual", dataPreviewRows: null, @@ -2253,7 +2266,7 @@ function ensureTableCreateDialog(manager) { tableCreateDialogState.dataTextarea.focus(); return; } - modal.requestClose("cancel"); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); }); tableCreateDialogState.createFromDataLink.addEventListener( @@ -2351,14 +2364,36 @@ function ensureTableCreateDialog(manager) { updateTableCreateDialogButtons(tableCreateDialogState); }); - modal.beforeClose = function (source) { - return confirmDiscardTableCreateChanges(tableCreateDialogState); - }; + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); dialog.addEventListener("close", function () { var state = tableCreateDialogState; clearTableCreateDialogError(state); setTableCreateDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } }); return tableCreateDialogState; @@ -2379,12 +2414,15 @@ function openTableCreateDialog(button, manager) { menu.open = false; } state.manager = manager; - + state.currentButton = button; + state.shouldRestoreFocus = true; state.title.textContent = "Create a table in " + data.databaseName; clearTableCreateDialogError(state); resetTableCreateDialog(state); loadTableCreateForeignKeyTargets(state); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } state.tableName.focus(); } @@ -2410,7 +2448,6 @@ function initTableCreateActions(manager) { function setRowDeleteDialogBusy(state, isBusy) { state.isBusy = isBusy; - state.modal.busy = isBusy; state.confirmButton.disabled = isBusy; state.cancelButton.disabled = isBusy; state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row"; @@ -2657,7 +2694,6 @@ function showTableAlterDialogError(state, message) { function setTableAlterDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.modal.busy = isSaving; state.cancelButton.disabled = isSaving; state.addColumnButton.disabled = isSaving; state.backButton.disabled = isSaving; @@ -3793,7 +3829,8 @@ async function applyTableAlterChanges(state, result) { result.columnTypeAssignments || [], tableUrl, ); - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (tableAlterResultRenamesTable(result) && tableUrl) { window.location.href = tableUrl; } else { @@ -3854,7 +3891,8 @@ async function dropTableFromAlterDialog(state) { if (!response.ok || (responseData && responseData.ok === false)) { throw rowMutationRequestError(response, responseData); } - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); window.location.href = tableAlterDatabaseUrl() || "/"; } catch (error) { setTableAlterDialogSaving(state, false); @@ -3890,6 +3928,27 @@ function confirmDiscardTableAlterChanges(state) { return window.confirm("Discard table changes?"); } +function closeTableAlterDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableAlterChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function closeTableAlterDialog(state) { + if (!state || state.isSaving) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + function ensureTableAlterDialog(manager) { if (tableAlterDialogState) { return tableAlterDialogState; @@ -3898,8 +3957,7 @@ function ensureTableAlterDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = TABLE_ALTER_DIALOG_ID; dialog.className = "table-alter-dialog"; dialog.setAttribute("aria-labelledby", "table-alter-title"); @@ -3909,7 +3967,7 @@ function ensureTableAlterDialog(manager) {
      -
      +
      - +
      - - - - + + + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); tableAlterDialogState = { - modal: modal, dialog: dialog, form: dialog.querySelector(".table-alter-form"), title: dialog.querySelector(".modal-title"), @@ -3955,6 +4012,8 @@ function ensureTableAlterDialog(manager) { dropButton: dialog.querySelector(".table-alter-drop"), cancelButton: dialog.querySelector(".table-alter-cancel"), saveButton: dialog.querySelector(".table-alter-save"), + currentButton: null, + shouldRestoreFocus: true, isSaving: false, initialSignature: "", originalTableName: "", @@ -3996,7 +4055,7 @@ function ensureTableAlterDialog(manager) { }); tableAlterDialogState.cancelButton.addEventListener("click", function () { - modal.requestClose("cancel"); + closeTableAlterDialog(tableAlterDialogState); }); tableAlterDialogState.dropButton.addEventListener("click", function () { @@ -4017,17 +4076,36 @@ function ensureTableAlterDialog(manager) { } }); - modal.beforeClose = function (source) { - return ( - source === "cancel" || - confirmDiscardTableAlterChanges(tableAlterDialogState) - ); - }; + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); dialog.addEventListener("close", function () { var state = tableAlterDialogState; clearTableAlterDialogError(state); setTableAlterDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } }); return tableAlterDialogState; @@ -4048,7 +4126,8 @@ function openTableAlterDialog(button, manager) { menu.open = false; } state.manager = manager; - + state.currentButton = button; + state.shouldRestoreFocus = true; state.title.textContent = "Alter table " + data.tableName; clearTableAlterDialogError(state); resetTableAlterDialog(state, data); @@ -4058,7 +4137,9 @@ function openTableAlterDialog(button, manager) { tableAlterForeignKeyTargetsUrl(), { filterByType: false }, ); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } var firstName = state.columnList.querySelector(".table-alter-column-name"); if (firstName) { firstName.focus(); @@ -4361,8 +4442,7 @@ function ensureRowDeleteDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = ROW_DELETE_DIALOG_ID; dialog.className = "row-delete-dialog"; dialog.setAttribute("aria-labelledby", "row-delete-title"); @@ -4374,14 +4454,13 @@ function ensureRowDeleteDialog(manager) {

      Delete row ?

      - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); rowDeleteDialogState = { - modal: modal, dialog: dialog, title: dialog.querySelector(".modal-title"), message: dialog.querySelector(".row-delete-message"), @@ -4394,10 +4473,21 @@ function ensureRowDeleteDialog(manager) { currentPkPath: null, manager: manager, isBusy: false, + shouldRestoreFocus: true, }; rowDeleteDialogState.cancelButton.addEventListener("click", function () { - modal.requestClose("cancel"); + if (!rowDeleteDialogState.isBusy) { + rowDeleteDialogState.shouldRestoreFocus = true; + dialog.close(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog && !rowDeleteDialogState.isBusy) { + rowDeleteDialogState.shouldRestoreFocus = true; + dialog.close(); + } }); dialog.addEventListener("keydown", function (ev) { @@ -4409,6 +4499,25 @@ function ensureRowDeleteDialog(manager) { if (!rowDeleteDialogState.isBusy) { rowDeleteDialogState.confirmButton.click(); } + return; + } + if (ev.key !== "Escape") { + return; + } + if (rowDeleteDialogState.isBusy) { + ev.preventDefault(); + return; + } + ev.preventDefault(); + rowDeleteDialogState.shouldRestoreFocus = true; + dialog.close(); + }); + + dialog.addEventListener("cancel", function (ev) { + if (rowDeleteDialogState.isBusy) { + ev.preventDefault(); + } else { + rowDeleteDialogState.shouldRestoreFocus = true; } }); @@ -4416,6 +4525,13 @@ function ensureRowDeleteDialog(manager) { var state = rowDeleteDialogState; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } }); rowDeleteDialogState.confirmButton.addEventListener( @@ -4442,7 +4558,8 @@ function ensureRowDeleteDialog(manager) { throw rowMutationRequestError(response, data); } if (data && data.redirect) { - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); location.href = data.redirect; return; } @@ -4454,7 +4571,8 @@ function ensureRowDeleteDialog(manager) { var statusMessage = state.currentPkPath ? "Deleted row " + state.currentPkPath + "." : "Deleted row."; - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); state.currentRow.remove(); showRowMutationStatus(state.manager, statusMessage, false); if (focusTarget && document.contains(focusTarget)) { @@ -4483,9 +4601,11 @@ function openRowDeleteDialog(button, manager) { } state.manager = manager; + state.currentButton = button; state.currentRow = row; state.currentDeleteUrl = rowDeleteUrl(row); state.currentPkPath = rowDisplayLabel(row); + state.shouldRestoreFocus = true; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); @@ -4497,7 +4617,9 @@ function openRowDeleteDialog(button, manager) { ); state.rowId.textContent = state.currentPkPath || "this row"; - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } state.confirmButton.focus(); } @@ -5572,7 +5694,6 @@ function setRowEditDialogLoading(state, isLoading) { function setRowEditDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.modal.busy = isSaving; updateRowEditDialogButtons(state); } @@ -5790,6 +5911,18 @@ function confirmDiscardRowEditChanges(state) { return window.confirm(message); } +function closeRowEditDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardRowEditChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + function setRowInsertDialogTitle(state) { var insertData = tableInsertData() || {}; var title = rowEditIsMultipleInsert(state) @@ -6615,6 +6748,38 @@ async function insertBulkPreviewRows(state) { } } +function scheduleCloseRowEditDialogIfConfirmed(state) { + // Fix for an issue in Safari where hitting Esc would show + // the confirm() prompt asking if state should be discarded + // but the Esc key press would then cancel that dialog too. + // Wait for keyup, then move the confirm() to a fresh timer tick. + if (!state || state.isSaving || state.isClosePending) { + return false; + } + if (!rowEditDialogHasChanges(state)) { + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; + } + state.isClosePending = true; + var closeAfterKeyup = function () { + if (!state.isClosePending) { + return; + } + state.isClosePending = false; + closeRowEditDialogIfConfirmed(state); + }; + var onKeyup = function (ev) { + if (ev.key !== "Escape") { + return; + } + document.removeEventListener("keyup", onKeyup, true); + setTimeout(closeAfterKeyup, 0); + }; + document.addEventListener("keyup", onKeyup, true); + return true; +} + function findDataRowElement(root, rowId) { var elements = root.querySelectorAll("[data-row]"); for (var i = 0; i < elements.length; i += 1) { @@ -6704,8 +6869,9 @@ async function saveRowEditDialog(state) { } var formValues = collectRowFormValues(state); if (state.mode === "edit" && !Object.keys(formValues).length) { + state.shouldRestoreFocus = true; hideRowMutationStatus(); - state.modal.close(); + state.dialog.close(); return; } var payload = @@ -6738,8 +6904,9 @@ async function saveRowEditDialog(state) { insertedRowData, insertData.primaryKeys || [], ); + state.shouldRestoreFocus = false; if (!insertedRowId) { - state.modal.close({ restoreFocus: false }); + state.dialog.close(); var missingIdStatus = showRowMutationStatus( state.manager, "Inserted row. Refresh the page to see it.", @@ -6755,7 +6922,7 @@ async function saveRowEditDialog(state) { try { insertedRow = await fetchUpdatedRowElement(state); } catch (_error) { - state.modal.close({ restoreFocus: false }); + state.dialog.close(); var refreshFailedStatus = showRowMutationStatus( state.manager, "Inserted row, but could not refresh the table row. Refresh the page to see it.", @@ -6770,7 +6937,7 @@ async function saveRowEditDialog(state) { rowTitleLabel(insertedRow), ); var addedRow = addInsertedRowToPage(insertedRow); - state.modal.close({ restoreFocus: false }); + state.dialog.close(); showRowMutationStatus(state.manager, insertedStatusMessage, false); if (addedRow) { var insertedFocusTarget = @@ -6779,7 +6946,7 @@ async function saveRowEditDialog(state) { insertedFocusTarget.focus(); } } else { - state.modal.close({ restoreFocus: false }); + state.dialog.close(); var filteredStatus = showRowMutationStatus( state.manager, "Inserted row. It does not match the current filters.", @@ -6791,7 +6958,8 @@ async function saveRowEditDialog(state) { } if (isRowPage()) { - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); location.reload(); return; } @@ -6827,7 +6995,8 @@ async function saveRowEditDialog(state) { ); } - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (focusTarget && document.contains(focusTarget)) { focusTarget.focus(); } @@ -6971,8 +7140,7 @@ function ensureRowEditDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = ROW_EDIT_DIALOG_ID; dialog.className = "row-edit-dialog"; dialog.setAttribute("aria-labelledby", "row-edit-title"); @@ -6984,8 +7152,8 @@ function ensureRowEditDialog(manager) {

      Loading row...

      -
      - @@ -7015,15 +7183,14 @@ function ensureRowEditDialog(manager) {
      - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); rowEditDialogState = { - modal: modal, dialog: dialog, form: dialog.querySelector(".row-edit-form"), title: dialog.querySelector(".modal-title"), @@ -7054,6 +7221,7 @@ function ensureRowEditDialog(manager) { singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), + currentButton: null, currentRow: null, currentRowId: null, currentPkPath: null, @@ -7081,7 +7249,9 @@ function ensureRowEditDialog(manager) { manager: manager, isLoading: false, isSaving: false, + isClosePending: false, hasLoaded: false, + shouldRestoreFocus: true, }; rowEditDialogState.form.addEventListener("submit", function (ev) { @@ -7101,7 +7271,10 @@ function ensureRowEditDialog(manager) { rowEditDialogState.bulkInsertTextarea.focus(); return; } - modal.requestClose("cancel"); + if (!rowEditDialogState.isSaving) { + rowEditDialogState.shouldRestoreFocus = true; + dialog.close(); + } }); rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { @@ -7220,17 +7393,31 @@ function ensureRowEditDialog(manager) { }, ); - modal.beforeClose = function (source) { - return ( - source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState) - ); - }; + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeRowEditDialogIfConfirmed(rowEditDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); + }); dialog.addEventListener("close", function () { var state = rowEditDialogState; var shouldReloadOnClose = state.shouldReloadOnClose; var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; + state.isClosePending = false; state.bulkInsertLiveValidationError = null; state.shouldReloadOnClose = false; state.redirectOnCloseUrl = null; @@ -7243,6 +7430,13 @@ function ensureRowEditDialog(manager) { destroyRowEditFields(state); setRowEditDialogLoading(state, false); setRowEditDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } if (shouldReloadOnClose) { if (redirectOnCloseUrl) { location.href = redirectOnCloseUrl; @@ -7267,6 +7461,7 @@ async function openRowEditDialog(button, manager) { state.manager = manager; state.mode = "edit"; + state.currentButton = button; state.currentRow = row; state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); @@ -7283,7 +7478,7 @@ async function openRowEditDialog(button, manager) { } else { state.form.removeAttribute("action"); } - + state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; var loadId = state.loadId; @@ -7302,7 +7497,9 @@ async function openRowEditDialog(button, manager) { state.summary.textContent = ""; syncRowEditInsertModeUi(state); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } state.cancelButton.focus(); try { @@ -7342,6 +7539,7 @@ function openRowInsertDialog(button, manager) { state.manager = manager; state.mode = "insert"; + state.currentButton = button; state.currentRow = null; state.currentRowId = null; state.currentPkPath = null; @@ -7356,7 +7554,7 @@ function openRowInsertDialog(button, manager) { state.shouldReloadOnClose = false; state.redirectOnCloseUrl = null; resetBulkInsertPreview(state); - + state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; @@ -7378,7 +7576,9 @@ function openRowInsertDialog(button, manager) { state.summary.textContent = ""; syncRowEditInsertModeUi(state); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } renderRowInsertFields(state, insertData); } diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..0e6e2c29 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,56 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === "object" && typeof module !== "undefined" + ? (module.exports = factory()) + : typeof define === "function" && define.amd + ? define(factory) + : (global.jsonFormatHighlight = factory()); +})(this, function () { + "use strict"; + + var defaultColors = { + keyColor: "dimgray", + numberColor: "lightskyblue", + stringColor: "lightcoral", + trueColor: "lightseagreen", + falseColor: "#f66578", + nullColor: "cornflowerblue", + }; + + function index(json, colorOptions) { + if (colorOptions === void 0) colorOptions = {}; + + if (!json) { + return; + } + if (typeof json !== "string") { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, "&").replace(//g, ">"); + return json.replace( + /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, + function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) + ? colors.trueColor + : /false/.test(match) + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; + } + return '' + match + ""; + }, + ); + } + + return index; +}); diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js index e94ee865..a386b1fc 100644 --- a/datasette/static/mobile-column-actions.js +++ b/datasette/static/mobile-column-actions.js @@ -66,8 +66,7 @@ function initMobileColumnActions(manager) { return; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.className = "mobile-column-actions-dialog"; dialog.id = MOBILE_COLUMN_DIALOG_ID; dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); @@ -76,13 +75,13 @@ function initMobileColumnActions(manager) { Column actions
      -
      +
      Tap a column to reveal actions. - +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); triggerButton.setAttribute("aria-haspopup", "dialog"); triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); @@ -92,6 +91,7 @@ function initMobileColumnActions(manager) { var listWrap = dialog.querySelector(".mobile-column-list"); var doneButton = dialog.querySelector(".mobile-column-actions-done"); var expandedSectionId = null; + var shouldRestoreFocus = true; function updateExpandedSection() { Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { @@ -128,7 +128,16 @@ function initMobileColumnActions(manager) { } function closeDialog(options) { - modal.close(options); + options = options || {}; + shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + triggerButton.setAttribute("aria-expanded", "false"); + if (shouldRestoreFocus) { + triggerButton.focus(); + } + } } function renderDialog() { @@ -157,8 +166,7 @@ function initMobileColumnActions(manager) { topActions.className = "mobile-column-top-actions"; var showAllColumns = document.createElement("a"); - showAllColumns.className = - "modal-btn modal-btn-ghost mobile-column-top-action"; + showAllColumns.className = "btn btn-ghost mobile-column-top-action"; showAllColumns.href = manager.columnActions.showAllColumnsUrl(); showAllColumns.textContent = "Show all columns"; @@ -257,7 +265,9 @@ function initMobileColumnActions(manager) { if (!renderDialog()) { return; } - modal.show({ returnFocusTo: triggerButton }); + if (!dialog.open) { + dialog.showModal(); + } triggerButton.setAttribute("aria-expanded", "true"); var focusTarget = dialog.querySelector(".mobile-column-top-action") || @@ -278,8 +288,22 @@ function initMobileColumnActions(manager) { closeDialog(); }); + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeDialog(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeDialog(); + }); + dialog.addEventListener("close", function () { triggerButton.setAttribute("aria-expanded", "false"); + if (shouldRestoreFocus) { + triggerButton.focus(); + } }); window.addEventListener("resize", function () { diff --git a/datasette/static/modal.js b/datasette/static/modal.js deleted file mode 100644 index ca9ab353..00000000 --- a/datasette/static/modal.js +++ /dev/null @@ -1,167 +0,0 @@ -// Shared lifecycle for native modal dialogs. -(() => { - class DatasetteModal extends HTMLElement { - constructor() { - super(); - this.beforeClose = null; - this._busy = false; - this._restoreFocus = true; - this._returnFocusTo = null; - this._escapeCleanup = null; - this._escapeTimer = null; - } - - static create() { - const modal = document.createElement("datasette-modal"); - modal.appendChild(document.createElement("dialog")); - return modal; - } - - get dialog() { - return this.querySelector(":scope > dialog"); - } - - get busy() { - return this._busy; - } - - set busy(value) { - this._busy = !!value; - if (this.dialog) { - this.dialog.setAttribute("aria-busy", String(this._busy)); - } - } - - connectedCallback() { - const dialog = this.dialog; - if (!dialog) return; - dialog.classList.add("datasette-modal"); - this._listeners?.abort(); - this._listeners = new AbortController(); - const options = { signal: this._listeners.signal }; - let backdropPointerDown = false; - const outside = (event) => { - const rect = dialog.getBoundingClientRect(); - return ( - event.target === dialog && - (event.clientX < rect.left || - event.clientX > rect.right || - event.clientY < rect.top || - event.clientY > rect.bottom) - ); - }; - dialog.addEventListener( - "pointerdown", - (event) => { - backdropPointerDown = outside(event); - }, - options, - ); - dialog.addEventListener( - "click", - (event) => { - if (backdropPointerDown && outside(event)) - this.requestClose("backdrop"); - backdropPointerDown = false; - }, - options, - ); - dialog.addEventListener( - "keydown", - (event) => { - if (event.key !== "Escape" || event.defaultPrevented) return; - // A nested native dialog or plugin picker gets first refusal. - if (event.target.closest("dialog") !== dialog) return; - event.preventDefault(); - if (this.busy || this._escapeCleanup || this._escapeTimer !== null) - return; - // Safari can otherwise use this Escape press to cancel confirm() too. - // Only keyboard dismissals wait for keyup; native cancel events needn't. - const onKeyup = (up) => { - if (up.key !== "Escape") return; - this._escapeCleanup(); - this._escapeCleanup = null; - this._escapeTimer = setTimeout(() => { - this._escapeTimer = null; - this.requestClose("escape"); - }, 0); - }; - this.ownerDocument.addEventListener("keyup", onKeyup, true); - this._escapeCleanup = () => - this.ownerDocument.removeEventListener("keyup", onKeyup, true); - }, - options, - ); - dialog.addEventListener( - "cancel", - (event) => { - if (event.target !== dialog) return; - event.preventDefault(); - if (!this._escapeCleanup && this._escapeTimer === null) - this.requestClose("escape"); - }, - options, - ); - dialog.addEventListener( - "close", - (event) => { - if (event.target !== dialog || dialog.open) return; - this._clearPendingClose(); - this.busy = false; - if (this._restoreFocus && this._returnFocusTo?.isConnected) { - // Menu actions may have become hidden while the dialog was open. - const details = this._returnFocusTo.closest("details:not([open])"); - const target = - details?.querySelector("summary") || this._returnFocusTo; - target.focus({ preventScroll: true }); - } - this._returnFocusTo = null; - }, - options, - ); - } - - disconnectedCallback() { - this._listeners?.abort(); - this._clearPendingClose(); - this._returnFocusTo = null; - if (this.dialog?.open) this.dialog.close(); - this.busy = false; - } - - _clearPendingClose() { - this._escapeCleanup?.(); - this._escapeCleanup = null; - clearTimeout(this._escapeTimer); - this._escapeTimer = null; - } - - show({ returnFocusTo, initialFocus } = {}) { - const dialog = this.dialog; - if (!dialog.open) { - this._clearPendingClose(); - this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement; - this._restoreFocus = true; - dialog.showModal(); - } - if (typeof initialFocus === "function") initialFocus(); - else initialFocus?.focus(); - } - - requestClose(source = "cancel") { - if (!this.dialog.open || this.busy) return false; - if (this.beforeClose && this.beforeClose(source) === false) return false; - this.close(); - return true; - } - - close({ restoreFocus = true } = {}) { - this._clearPendingClose(); - this._restoreFocus = restoreFocus; - this.dialog.close(); - } - } - - customElements.define("datasette-modal", DatasetteModal); - window.DatasetteModal = DatasetteModal; -})(); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index df2516f9..ec2d23d8 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -10,22 +10,277 @@ class NavigationSearch extends HTMLElement { this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; this.statusId = `navigation-search-status-${this.instanceId}`; this.titleId = `navigation-search-title-${this.instanceId}`; + this.attachShadow({ mode: "open" }); this.selectedIndex = -1; this.matches = []; this.renderedMatches = []; this.debounceTimer = null; - } + this.restoreFocusTarget = null; + this.shouldRestoreFocus = true; - connectedCallback() { - if (this._initialized) return; - this._initialized = true; this.render(); this.setupEventListeners(); } render() { - this.innerHTML = ` - + this.shadowRoot.innerHTML = ` + + +
      @@ -47,22 +302,23 @@ class NavigationSearch extends HTMLElement { >
      -
      +
      ↑ ↓ Navigate Enter Select Esc Close
      - + `; } setupEventListeners() { - const dialog = this.querySelector("dialog"); - const input = this.querySelector(".search-input"); - const closeButton = this.querySelector(".close-search"); - const resultsContainer = this.querySelector(".results-container"); + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); + const closeButton = this.shadowRoot.querySelector(".close-search"); + const resultsContainer = + this.shadowRoot.querySelector(".results-container"); // Global keyboard listener for "/" document.addEventListener("keydown", (e) => { @@ -99,6 +355,8 @@ class NavigationSearch extends HTMLElement { } else if (e.key === "Enter") { e.preventDefault(); this.selectCurrentItem(); + } else if (e.key === "Escape") { + this.closeMenu(); } }); @@ -122,6 +380,18 @@ class NavigationSearch extends HTMLElement { } }); + // Close on backdrop click + dialog.addEventListener("click", (e) => { + if (e.target === dialog) { + this.closeMenu(); + } + }); + + dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this.closeMenu(); + }); + dialog.addEventListener("close", () => { this.onMenuClosed(); }); @@ -162,6 +432,19 @@ class NavigationSearch extends HTMLElement { } } + focusRestoreTarget(trigger) { + if (trigger && typeof trigger.focus === "function") { + return trigger; + } + if ( + document.activeElement && + typeof document.activeElement.focus === "function" + ) { + return document.activeElement; + } + return null; + } + setNavigationTriggersExpanded(expanded) { if (typeof document.querySelectorAll !== "function") { return; @@ -182,8 +465,8 @@ class NavigationSearch extends HTMLElement { } updateComboboxState() { - const dialog = this.querySelector("dialog"); - const input = this.querySelector(".search-input"); + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); const matches = this.renderedMatches || []; this.setElementAttribute( input, @@ -208,7 +491,7 @@ class NavigationSearch extends HTMLElement { } setStatus(message) { - const status = this.querySelector(`#${this.statusId}`); + const status = this.shadowRoot.querySelector(`#${this.statusId}`); if (status) { status.textContent = message || ""; } @@ -418,7 +701,7 @@ class NavigationSearch extends HTMLElement { section.render(node, { navigationSearch: this, container, - input: this.querySelector(".search-input"), + input: this.shadowRoot.querySelector(".search-input"), }); }); } @@ -457,8 +740,8 @@ class NavigationSearch extends HTMLElement { } renderResults() { - const container = this.querySelector(".results-container"); - const input = this.querySelector(".search-input"); + const container = this.shadowRoot.querySelector(".results-container"); + const input = this.shadowRoot.querySelector(".search-input"); const showStartContent = !input.value.trim(); const jumpSections = showStartContent ? this.jumpSections() : []; const startBlock = showStartContent @@ -570,15 +853,18 @@ class NavigationSearch extends HTMLElement { } } - openMenu(returnFocusTo) { - const input = this.querySelector(".search-input"); + openMenu(trigger) { + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); - this.querySelector("datasette-modal").show({ - returnFocusTo, - initialFocus: input, - }); + this.restoreFocusTarget = this.focusRestoreTarget(trigger); + this.shouldRestoreFocus = true; + if (!dialog.open) { + dialog.showModal(); + } this.setNavigationTriggersExpanded(true); input.value = ""; + input.focus(); // Reset state, then populate the default jump list. this.matches = []; @@ -588,15 +874,29 @@ class NavigationSearch extends HTMLElement { } closeMenu(options = {}) { - this.querySelector("datasette-modal").close(options); + const dialog = this.shadowRoot.querySelector("dialog"); + this.shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + this.onMenuClosed(); + } } onMenuClosed() { - const input = this.querySelector(".search-input"); + const input = this.shadowRoot.querySelector(".search-input"); this.setElementAttribute(input, "aria-expanded", "false"); this.removeElementAttribute(input, "aria-activedescendant"); this.setNavigationTriggersExpanded(false); this.setStatus(""); + if ( + this.shouldRestoreFocus && + this.restoreFocusTarget && + typeof this.restoreFocusTarget.focus === "function" + ) { + this.restoreFocusTarget.focus(); + } + this.restoreFocusTarget = null; } escapeHtml(text) { diff --git a/datasette/static/table.js b/datasette/static/table.js index 6b3e619c..74a96d8e 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -157,7 +157,6 @@ function createSetColumnTypeOption(value, name, description, checked) { function setSetColumnTypeDialogBusy(state, isBusy) { state.isBusy = isBusy; - state.modal.busy = isBusy; state.saveButton.disabled = isBusy; state.cancelButton.disabled = isBusy; Array.from( @@ -186,8 +185,7 @@ function ensureSetColumnTypeDialog() { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = SET_COLUMN_TYPE_DIALOG_ID; dialog.className = "set-column-type-dialog"; dialog.setAttribute("aria-labelledby", "set-column-type-title"); @@ -198,17 +196,16 @@ function ensureSetColumnTypeDialog() {

      -
      +
      - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); setColumnTypeDialogState = { - modal: modal, dialog: dialog, meta: dialog.querySelector(".modal-meta"), status: dialog.querySelector(".set-column-type-status"), @@ -223,7 +220,21 @@ function ensureSetColumnTypeDialog() { }; setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - modal.requestClose("cancel"); + if (!setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + if (setColumnTypeDialogState.isBusy) { + ev.preventDefault(); + } }); dialog.addEventListener("close", function () { @@ -231,52 +242,49 @@ function ensureSetColumnTypeDialog() { setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); }); - setColumnTypeDialogState.saveButton.addEventListener( - "click", - async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; + setColumnTypeDialogState.saveButton.addEventListener("click", async function () { + var state = setColumnTypeDialogState; + var selected = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + var selectedType = selected ? selected.value : ""; + var currentType = state.currentConfig.current + ? state.currentConfig.current.type + : ""; - if (selectedType === currentType) { - state.modal.close(); - return; + if (selectedType === currentType) { + state.dialog.close(); + return; + } + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, true); + + var payload = { + column: state.currentColumn, + column_type: selectedType ? { type: selectedType } : null, + }; + + try { + var response = await fetch(getSetColumnTypeData().path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + var message = (data.errors || ["Request failed"]).join(" "); + throw new Error(message); } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); - } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }, - ); + location.reload(); + } catch (error) { + setSetColumnTypeDialogBusy(state, false); + showSetColumnTypeDialogError(state, error.message || "Request failed"); + } + }); return setColumnTypeDialogState; } @@ -333,7 +341,9 @@ function openSetColumnTypeDialog(th) { state.optionsWrap.appendChild(emptyState); } - state.modal.show(); + if (!state.dialog.open) { + state.dialog.showModal(); + } var selectedOption = state.dialog.querySelector( 'input[name="set-column-type-choice"]:checked', ); @@ -357,10 +367,9 @@ function shouldShowShowAllColumns() { function hasMultipleVisibleColumns(manager) { return ( - Array.from( - document.querySelectorAll(manager.selectors.tableHeaders), - ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1") - .length > 1 + Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( + (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", + ).length > 1 ); } @@ -640,12 +649,10 @@ function filterRowNumberFromName(name) { } function nextFilterRowNumber(manager) { - return ( - filterRowsWithControls(manager).reduce((max, row) => { - var column = row.querySelector("select"); - return Math.max(max, filterRowNumberFromName(column && column.name)); - }, 0) + 1 - ); + return filterRowsWithControls(manager).reduce((max, row) => { + var column = row.querySelector("select"); + return Math.max(max, filterRowNumberFromName(column && column.name)); + }, 0) + 1; } function setFilterRowNumber(row, number) { @@ -672,11 +679,9 @@ function updateFilterRowButtons(manager) { if (addButton) { addButton.hidden = index !== rows.length - 1 || !column.value; } - var visibleButtonCount = [removeButton, addButton].filter( - function (button) { - return button && !button.hidden; - }, - ).length; + var visibleButtonCount = [removeButton, addButton].filter(function (button) { + return button && !button.hidden; + }).length; row.classList.toggle( "filter-controls-row-has-buttons", visibleButtonCount > 0, @@ -698,9 +703,7 @@ function cloneFilterRow(row) { clone.querySelector(".filter-op select").name = "_filter_op"; clone.querySelector("input.filter-value").name = "_filter_value"; resetFilterRow(clone); - clone - .querySelectorAll(".filter-row-icon") - .forEach((button) => button.remove()); + clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove()); return clone; } @@ -857,45 +860,10 @@ function openColumnChooser() { }); } -function initCountAll() { - var button = document.querySelector(".count-all"); - if (!button) { - return; - } - button.addEventListener("click", async function () { - var count = document.querySelector(".table-count"); - var error = document.querySelector(".count-error"); - button.disabled = true; - button.textContent = "Counting…"; - error.textContent = ""; - try { - var response = await fetch(button.dataset.countUrl + location.search, { - method: "POST", - headers: { - Accept: "application/json", - }, - }); - var data = await response.json(); - if (!response.ok || !data.ok) { - throw new Error((data.errors || ["Count failed"]).join(" ")); - } - count.textContent = - data.count.toLocaleString("en-US") + - (data.count === 1 ? " row" : " rows"); - button.remove(); - } catch (ex) { - error.textContent = ex.message || "Count failed"; - button.disabled = false; - button.textContent = "count all"; - } - }); -} - // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; - initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/telemetry.py b/datasette/telemetry.py deleted file mode 100644 index 2fd5a9dc..00000000 --- a/datasette/telemetry.py +++ /dev/null @@ -1,481 +0,0 @@ -""" -OpenTelemetry integration for Datasette. - -This uses `opentelemetry-api` only. Providers, exporters and sampling are -configured by whoever runs Datasette, for example `opentelemetry-instrument`. -""" - -import contextvars -import re -import threading -import time -import weakref -from contextlib import contextmanager - -from opentelemetry import context as otel_context_api -from opentelemetry import metrics as otel_metrics -from opentelemetry import trace as otel_trace -from opentelemetry.propagate import extract -from opentelemetry.propagators.textmap import Getter -from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span - -from .telemetry_registry import ( - DB_NAMESPACE, - DB_SYSTEM, - ERROR_TYPE, - HTTP_REQUEST_METHOD, - HTTP_RESPONSE_STATUS_CODE, - INTERNAL_CLIENT, - M_CONNECTIONS_OPEN, - M_OPERATION_DURATION, - M_QUERIES_INTERRUPTED, - M_QUERIES_PENDING, - M_THREADS_LIMIT, - M_THREADS_QUEUE_DEPTH, - M_WRITE_QUEUE_DEPTH, - M_WRITE_QUEUE_WAIT, - OPERATION, - SERVER_ADDRESS, - URL_PATH, - URL_SCHEME, - USER_AGENT_ORIGINAL, -) -from .version import __version__ - -# True while code is executing within a datasette.client request. Defined -# here rather than in app.py to avoid a circular import. -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) - -# The semantic conventions version matching the attribute names used here. -# 1.30.0 renamed `db.system` to `db.system.name`, so update this when -# renaming attributes to match a newer version. -SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0" - -tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL) -meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL) - -MAX_SQL_LENGTH = 2048 - - -def sql_attribute(sql: str) -> str: - "Truncate SQL text so it is safe to attach to a span as an attribute." - sql = sql.strip() - if len(sql) <= MAX_SQL_LENGTH: - return sql - return sql[:MAX_SQL_LENGTH] + "…[truncated]" - - -def callback_name(fn) -> str: - """ - The name recorded as `datasette.callback` for a callback-style call. - - Falls back to the type name for callables such as `functools.partial` - that have no `__qualname__`. - """ - return getattr(fn, "__qualname__", type(fn).__name__) - - -def linked_root_span_kwargs(context=None): - """ - Keyword arguments that start a new root span with a ``Link`` back to - the current span. - - Use this for work that can outlive the span that caused it, such as a - background task or a ``block=False`` write. - - Pass ``context`` to link to the span in a previously captured context - instead of the current one. If there is no valid span, no link is added. - - Works with any tracer:: - - with my_tracer.start_as_current_span( - "myplugin.job", **linked_root_span_kwargs() - ): - ... - """ - cause = get_current_span(context).get_span_context() - links = [Link(cause)] if cause.is_valid else [] - return {"context": otel_context_api.Context(), "links": links} - - -# Keywords that can be recorded as db.operation.name. SQL can be supplied by -# users, so an allowlist keeps the number of distinct values small. -DB_OPERATION_ALLOWLIST = frozenset( - { - "SELECT", - "INSERT", - "UPDATE", - "DELETE", - "CREATE", - "DROP", - "ALTER", - "PRAGMA", - "EXPLAIN", - "REPLACE", - "VACUUM", - "ANALYZE", - "WITH", - } -) - -_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)") - - -def sql_operation_name(sql: str) -> str | None: - """ - The statement's leading keyword if it is in the allowlist, else None. - - Statements that start with a comment or "(" return None. Statements - starting with a CTE return `WITH`. Only call this for a single statement. - """ - match = _LEADING_KEYWORD.match(sql) - if not match: - return None - keyword = match.group(1).upper() - if keyword in DB_OPERATION_ALLOWLIST: - return keyword - return None - - -# --- The HTTP request span ------------------------------------------------ - - -class _ScopeHeadersGetter(Getter): - "Read W3C trace context from an ASGI scope's headers." - - def get(self, carrier, key): - wanted = key.lower().encode("latin-1") - values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted] - return values or None - - def keys(self, carrier): - return [k.decode("latin-1") for k, _ in carrier] - - -_HEADERS_GETTER = _ScopeHeadersGetter() - - -# Methods defined by RFC 9110 plus PATCH (RFC 5789). Anything else is -# recorded as `_OTHER`, as recommended by semantic conventions. -_KNOWN_METHODS = frozenset( - {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"} -) - - -def clamp_http_method(method): - "The request method if it is one we recognise, else ``_OTHER``." - method = (method or "").upper() - return method if method in _KNOWN_METHODS else "_OTHER" - - -def _first_header(headers, name): - "The first value of a header, decoded, or None." - for key, value in headers: - if key.lower() == name: - return value.decode("latin-1") - return None - - -def _url_path(scope): - """ - The request path, with any query string removed. - - Prefers `raw_path`, which preserves encoded slashes in database and - table names. Some clients include the query string in `raw_path`, so - that is stripped as well. - """ - raw_path = scope.get("raw_path") - if raw_path: - if isinstance(raw_path, bytes): - raw_path = raw_path.decode("latin-1") - return raw_path.split("?", 1)[0] - return scope.get("path", "") - - -# The request span is passed to the router in the ASGI scope, because a -# plugin's asgi_wrapper() middleware may have made its own span current. -# Absent if the span is not recording. -REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span" - - -def request_span(scope): - """ - The recording request span for an ASGI scope, or None. - - Falls back to the current span, for when Datasette is running under - other instrumentation. - """ - span = scope.get(REQUEST_SPAN_SCOPE_KEY) - if span is None: - span = otel_trace.get_current_span() - return span if span.is_recording() else None - - -class TelemetryMiddleware: - """ - One `SpanKind.SERVER` span per HTTP request. - - The span ends after the full response, including any streamed body, - has been sent. - """ - - def __init__(self, app): - self.app = app - - async def __call__(self, scope, receive, send): - # Pass lifespan and websocket scopes straight through - if scope["type"] != "http": - await self.app(scope, receive, send) - return - headers = scope.get("headers") or [] - # Uses the global propagator, configured with OTEL_PROPAGATORS - context = extract(headers, getter=_HEADERS_GETTER) - method = clamp_http_method(scope.get("method", "")) - # Renamed to include the route once routing has happened - with tracer.start_as_current_span( - method, context=context, kind=SpanKind.SERVER - ) as span: - if not span.is_recording(): - # No provider installed, or the trace was not sampled - await self.app(scope, receive, send) - return - span.set_attribute(HTTP_REQUEST_METHOD, method) - span.set_attribute(URL_PATH, _url_path(scope)) - scheme = scope.get("scheme") - if scheme: - span.set_attribute(URL_SCHEME, scheme) - host = _first_header(headers, b"host") - if host: - span.set_attribute(SERVER_ADDRESS, host) - user_agent = _first_header(headers, b"user-agent") - if user_agent: - span.set_attribute(USER_AGENT_ORIGINAL, user_agent) - if _in_datasette_client.get(): - span.set_attribute(INTERNAL_CLIENT, True) - - scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span}) - - # Some responses are sent without a Response object, so the - # status is captured by wrapping send() - status_holder = {} - - async def wrapped_send(message): - if ( - message["type"] == "http.response.start" - and "status" not in status_holder - ): - status_holder["status"] = message["status"] - await send(message) - - escaped = False - try: - await self.app(scope, receive, wrapped_send) - except BaseException as exception: - # Includes asyncio.CancelledError when a client disconnects - escaped = True - span.set_attribute(ERROR_TYPE, type(exception).__name__) - span.set_status(Status(StatusCode.ERROR, str(exception))) - raise - finally: - status = status_holder.get("status") - if status is not None: - span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status) - # 4xx responses are not errors for a server span. If an - # exception escaped, keep its class name as error.type. - if status >= 500 and not escaped: - span.set_status(Status(StatusCode.ERROR)) - span.set_attribute(ERROR_TYPE, str(status)) - - -# --- Metrics -------------------------------------------------------------- - - -def _duration_attributes(database_name, operation): - return { - DB_SYSTEM: "sqlite", - DB_NAMESPACE: database_name, - OPERATION: operation, - } - - -# Instruments use plain text descriptions. The registry entries have longer -# reStructuredText descriptions for the documentation. - -sql_operation_duration = meter.create_histogram( - M_OPERATION_DURATION, - unit=M_OPERATION_DURATION.unit, - description="Duration of a SQL operation issued by Datasette", - explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets, -) - -write_queue_wait = meter.create_histogram( - M_WRITE_QUEUE_WAIT, - unit=M_WRITE_QUEUE_WAIT.unit, - description=( - "Time a write spent queued behind the single write thread for its database" - ), - explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets, -) - -queries_interrupted = meter.create_counter( - M_QUERIES_INTERRUPTED, - unit=M_QUERIES_INTERRUPTED.unit, - description="Queries cancelled for exceeding sql_time_limit_ms", -) - - -@contextmanager -def record_operation_duration(database_name, operation): - """ - Record `db.client.operation.duration` for one SQL operation. - - Sets `error.type` to the exception class on failure. For a `block=False` - write this measures the time taken to enqueue the write. - """ - attributes = _duration_attributes(database_name, operation) - started = time.perf_counter() - try: - yield - except BaseException as exception: - attributes[ERROR_TYPE] = type(exception).__qualname__ - raise - finally: - sql_operation_duration.record(time.perf_counter() - started, attributes) - - -def record_write_queue_wait(database_name, waited_ns): - write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name}) - - -def record_query_interrupted(database_name): - queries_interrupted.add(1, {DB_NAMESPACE: database_name}) - - -# Live Datasette instances reported by the gauges below. The lock is needed -# because gauge callbacks run on the SDK's collection thread. -# -# The pool gauges do not identify which instance they came from, so they -# are only meaningful for a process running a single Datasette instance. -_live_datasettes = weakref.WeakSet() -_live_datasettes_lock = threading.Lock() - - -def register_datasette(ds): - "Start reporting pool/queue gauges for this Datasette instance." - with _live_datasettes_lock: - _live_datasettes.add(ds) - - -def unregister_datasette(ds): - "Stop reporting gauges for an instance that has been closed." - with _live_datasettes_lock: - _live_datasettes.discard(ds) - - -def _live_instances(): - with _live_datasettes_lock: - return list(_live_datasettes) - - -def _databases_of(ds): - "Every Database attached to an instance, including the internal database." - databases = list(ds.databases.values()) - internal = getattr(ds, "_internal_database", None) - if internal is not None: - databases.append(internal) - return databases - - -def observe_sql_thread_limit(options=None): - "Size of the shared read-query thread pool (the num_sql_threads setting)." - for ds in _live_instances(): - if ds.executor is None: - # num_sql_threads=0 - queries run on the event loop, no pool. - continue - yield otel_metrics.Observation(ds.setting("num_sql_threads"), {}) - - -def observe_sql_thread_queue_depth(options=None): - """ - Read queries waiting for a free thread in the shared pool. - - `_work_queue` is a private attribute of ThreadPoolExecutor, so this - reports nothing if it is missing. - """ - for ds in _live_instances(): - if ds.executor is None: - continue - work_queue = getattr(ds.executor, "_work_queue", None) - if work_queue is None: - continue - yield otel_metrics.Observation(work_queue.qsize(), {}) - - -def observe_pending_queries(options=None): - """ - Read queries submitted to the pool and not yet finished, per database. - - Reads `len()` without `_pending_execute_futures_lock` to avoid blocking - queries. - """ - for ds in _live_instances(): - for db in _databases_of(ds): - yield otel_metrics.Observation( - len(db._pending_execute_futures), {DB_NAMESPACE: db.name} - ) - - -def observe_write_queue_depth(options=None): - "Writes queued behind the single write thread, per database." - for ds in _live_instances(): - for db in _databases_of(ds): - write_queue = db._write_queue - if write_queue is None: - # No write has ever been queued for this database. - continue - yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name}) - - -def observe_open_connections(options=None): - "Open SQLite connections tracked for closing, per database." - for ds in _live_instances(): - for db in _databases_of(ds): - yield otel_metrics.Observation( - len(db._all_connections), {DB_NAMESPACE: db.name} - ) - - -sql_thread_limit_gauge = meter.create_observable_gauge( - M_THREADS_LIMIT, - callbacks=[observe_sql_thread_limit], - unit=M_THREADS_LIMIT.unit, - description="Maximum concurrent read queries (the num_sql_threads setting)", -) - -sql_thread_queue_depth_gauge = meter.create_observable_gauge( - M_THREADS_QUEUE_DEPTH, - callbacks=[observe_sql_thread_queue_depth], - unit=M_THREADS_QUEUE_DEPTH.unit, - description="Read queries waiting for a free thread in the shared SQL pool", -) - -pending_queries_gauge = meter.create_observable_gauge( - M_QUERIES_PENDING, - callbacks=[observe_pending_queries], - unit=M_QUERIES_PENDING.unit, - description="Read queries submitted to the pool and not yet complete", -) - -write_queue_depth_gauge = meter.create_observable_gauge( - M_WRITE_QUEUE_DEPTH, - callbacks=[observe_write_queue_depth], - unit=M_WRITE_QUEUE_DEPTH.unit, - description="Writes queued behind a database's single write thread", -) - -open_connections_gauge = meter.create_observable_gauge( - M_CONNECTIONS_OPEN, - callbacks=[observe_open_connections], - unit=M_CONNECTIONS_OPEN.unit, - description="Open SQLite connections tracked for closing", -) diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py deleted file mode 100644 index 63d5b89b..00000000 --- a/datasette/telemetry_registry.py +++ /dev/null @@ -1,502 +0,0 @@ -""" -Every span, metric and attribute that Datasette emits. - -These entries are used by the instrumentation code, by `docs/telemetry_doc.py` -to generate the documentation, and by `tests/test_telemetry_registry.py` to -check that the emitted telemetry matches the registry. -""" - -from opentelemetry.trace import SpanKind - - -class Attribute(str): - """ - A span attribute key, carrying its own documentation. - - Subclasses `str` so it can be handed straight to `set_attribute()`. - - Part of Datasette's public plugin API - plugins declare their own - telemetry registries with these classes. See the "Telemetry for plugin - authors" documentation. - """ - - __slots__ = ("description", "optional", "values") - - def __new__(cls, name, description, optional=False, values=None): - self = super().__new__(cls, name) - self.description = description - self.optional = optional - # The allowed values for this attribute, or None to allow any value - self.values = frozenset(values) if values is not None else None - return self - - def __reduce__(self): - # Copies and pickles become a plain str, since __new__ requires the - # extra arguments. ConsoleMetricExporter deepcopies attribute keys. - return (str, (str(self),)) - - def __repr__(self): - return f"Attribute({str(self)!r})" - - -class SpanName(str): - """A span name, carrying its documentation and the attributes it may set. - - Part of Datasette's public plugin API, like `Attribute`. - """ - - __slots__ = ("attributes", "description", "dynamic", "kind", "prefix") - - def __new__( - cls, - name, - description, - attributes=(), - prefix=False, - dynamic=False, - kind=SpanKind.INTERNAL, - ): - self = super().__new__(cls, name) - self.description = description - self.attributes = tuple(attributes) - # Match emitted names that start with this prefix, for names with a - # variable suffix such as SpanName("chat ", ..., prefix=True) - self.prefix = prefix - # The emitted name is built at runtime, so `span_for()` matches it by - # span kind. The entry's string is a template for the documentation. - self.dynamic = dynamic - self.kind = kind - return self - - def __reduce__(self): - # See Attribute.__reduce__. - return (str, (str(self),)) - - def __repr__(self): - return f"SpanName({str(self)!r})" - - -class MetricName(str): - "A metric name, carrying its instrument kind, unit and attributes." - - __slots__ = ("attributes", "buckets", "description", "kind", "unit") - - def __new__(cls, name, kind, unit, description, attributes=(), buckets=None): - self = super().__new__(cls, name) - self.kind = kind - self.unit = unit - self.description = description - self.attributes = tuple(attributes) - # Explicit bucket boundaries, for histograms only - self.buckets = tuple(buckets) if buckets is not None else None - return self - - def __reduce__(self): - # See Attribute.__reduce__. - return (str, (str(self),)) - - def __repr__(self): - return f"MetricName({str(self)!r})" - - -COUNTER = "Counter" -UPDOWN_COUNTER = "UpDownCounter" -HISTOGRAM = "Histogram" -GAUGE = "Observable gauge" - - -# --- Attributes ----------------------------------------------------------- - -HTTP_REQUEST_METHOD = Attribute( - "http.request.method", - "The HTTP request method. Methods outside the nine defined by RFC 9110 " - "and RFC 5789 are recorded as ``_OTHER``.", -) -HTTP_RESPONSE_STATUS_CODE = Attribute( - "http.response.status_code", - "The HTTP response status code. Omitted if no response was started.", - optional=True, -) -HTTP_ROUTE = Attribute( - "http.route", - "The regular expression for the matched route, for example " - "``/(?P[^\\/\\.]+)/(?P
      [^\\/\\.]+)(\\.(?P\\w+))?$`` " - "for a table page. Use this attribute to group requests by route. " - "Omitted when no route matches.", - optional=True, -) -URL_PATH = Attribute( - "url.path", - "The URL path, excluding the query string.", -) -URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.") -SERVER_ADDRESS = Attribute( - "server.address", - "The ``Host`` header, including any ``:port`` suffix. This value is " - "supplied by the client.", - optional=True, -) -USER_AGENT_ORIGINAL = Attribute( - "user_agent.original", - "The ``User-Agent`` header, verbatim. Omitted if the client sent none.", - optional=True, -) -INTERNAL_CLIENT = Attribute( - "datasette.internal_client", - "``True`` for requests made through ``datasette.client``. Calls made " - "inside another request produce a nested ``SERVER`` span. Filter on " - "this attribute to exclude internal requests from request counts. " - "Omitted for requests received over the network.", - optional=True, -) -ERROR_TYPE = Attribute( - "error.type", - "The exception class name for a failed operation. On HTTP spans, also " - "set to the status code as a string for 5xx responses. A 4xx response " - "alone does not set this attribute or an error status.", - optional=True, -) - -DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") -DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") -OPERATION = Attribute( - "datasette.operation", - "Whether the operation was a read or a write.", - values={"read", "write"}, -) -DB_QUERY_TEXT = Attribute( - "db.query.text", - "The SQL, truncated to 2048 characters. Bound parameter values are not " - "recorded. For callback methods, ``datasette.callback`` is recorded instead.", - optional=True, -) -CALLBACK = Attribute( - "datasette.callback", - "The qualified name of the Python callable passed to ``execute_fn()``, " - "``execute_write_fn()`` or ``execute_isolated_fn()``, for example " - "``TableInsertView.post..insert_or_upsert_rows``. Set instead of " - "``db.query.text``. Lambdas appear as ````; use a named function " - "for a more descriptive span.", - optional=True, -) -DB_OPERATION_NAME = Attribute( - "db.operation.name", - "The statement's leading keyword, such as ``SELECT``, ``INSERT`` or " - "``CREATE``, if it matches the supported allowlist. Statements beginning " - "with a common table expression report ``WITH``. Omitted for unrecognized " - "keywords and ``execute_write_script()``.", - optional=True, -) -PARAM_COUNT = Attribute( - "datasette.param_count", - "Number of bound parameters. Recorded instead of the values themselves.", - optional=True, -) -PARAM_SETS = Attribute( - "datasette.param_sets", - "Number of parameter sets consumed by ``execute_write_many()``. " - "The parameter values are not recorded.", - optional=True, -) -TIME_LIMIT_MS = Attribute( - "datasette.time_limit_ms", - "Time limit applied to the read query, in milliseconds: " - ":ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.", - optional=True, -) -ROWS_RETURNED = Attribute( - "datasette.rows_returned", - "Number of rows returned by a successful read query.", - optional=True, -) -TRUNCATED = Attribute( - "datasette.truncated", - "True if the result was cut short by :ref:`setting_max_returned_rows`.", - optional=True, -) -INTERRUPTED = Attribute( - "datasette.interrupted", - "True if the query exceeded its time limit. The span status is set to " - "``ERROR`` unless the caller used a ``custom_time_limit`` shorter than " - ":ref:`setting_sql_time_limit_ms`, in which case the status is left unset.", - optional=True, -) -SQL_ERROR_SUPPRESSED = Attribute( - "datasette.sql_error_suppressed", - "True for a non-timeout SQL error with ``log_sql_errors=False``. The " - "exception is still raised, but the span status is left unset.", - optional=True, -) -EXECUTESCRIPT = Attribute( - "datasette.executescript", - "True for ``execute_write_script()``, which runs multiple statements.", - optional=True, -) -EXECUTEMANY = Attribute( - "datasette.executemany", - "True for ``execute_write_many()``, which runs one statement against many " - "parameter sets.", - optional=True, -) -ISOLATED_CONNECTION = Attribute( - "datasette.isolated_connection", - "True if the write ran on its own connection rather than the shared write " - "connection.", -) -TRANSACTION = Attribute( - "datasette.transaction", - "False for statements such as ``VACUUM`` that cannot run inside a transaction.", -) - - -# --- Spans ---------------------------------------------------------------- - -HTTP_REQUEST = SpanName( - "{http.request.method} {http.route}", - "One span per HTTP request, containing spans from plugin middleware and " - "database operations. Named for the HTTP method and matched route, or " - "just the method if no route matches. Incoming ``traceparent`` headers " - "are extracted using the global propagator to continue the caller's " - "trace. Incoming ``baggage`` is not propagated into plugin or downstream " - "context in this release. Set ``OTEL_PROPAGATORS=none`` to disable " - "extraction. For public instances, strip trace context headers at your " - "proxy if callers should not supply trace context.", - ( - HTTP_REQUEST_METHOD, - HTTP_ROUTE, - URL_PATH, - URL_SCHEME, - SERVER_ADDRESS, - USER_AGENT_ORIGINAL, - HTTP_RESPONSE_STATUS_CODE, - ERROR_TYPE, - INTERNAL_CLIENT, - ), - dynamic=True, - kind=SpanKind.SERVER, -) - -DB_QUERY = SpanName( - "db.query", - "A SQL operation, including time spent queued for a worker thread. For " - "``block=False`` writes, the span ends after the write is queued. " - "Callback methods record ``datasette.callback`` in place of ``db.query.text``.", - ( - DB_SYSTEM, - DB_NAMESPACE, - DB_QUERY_TEXT, - CALLBACK, - DB_OPERATION_NAME, - PARAM_COUNT, - PARAM_SETS, - TIME_LIMIT_MS, - ROWS_RETURNED, - TRUNCATED, - INTERRUPTED, - SQL_ERROR_SUPPRESSED, - EXECUTESCRIPT, - EXECUTEMANY, - ), - kind=SpanKind.CLIENT, -) - -DB_QUERY_EXECUTE = SpanName( - "db.query.execute", - "The read executing inside a SQL worker thread. Child of ``db.query``; the " - "gap between the two is time spent waiting for a thread.", -) - -DB_WRITE_QUEUE_WAIT = SpanName( - "db.write.queue_wait", - "Time a write spent waiting in its database's write queue. For " - "``block=True``, this is a child of ``db.query``. For ``block=False``, " - "it is a root span linked to the span that queued the write, since the " - "write can outlive that request.", -) - -DB_WRITE_EXECUTE = SpanName( - "db.write.execute", - "The write executing on the write thread. For ``block=True``, this is " - "a child of ``db.query``. For ``block=False``, it is a root span linked " - "to the span that queued the write.", - (ISOLATED_CONNECTION, TRANSACTION), -) - -STARTUP = SpanName( - "datasette.startup", - "Startup work performed by ``invoke_startup()``, including registration " - "hooks, schema catalog updates, saved queries, column type configuration " - "and the ``startup`` hook. Runs during instance startup, either before " - "serving requests or as part of the first request.", -) - -SPANS = ( - HTTP_REQUEST, - DB_QUERY, - DB_QUERY_EXECUTE, - DB_WRITE_QUEUE_WAIT, - DB_WRITE_EXECUTE, - STARTUP, -) - - -def span_for(emitted_name, kind=None, spans=None): - """ - Resolve an emitted span name to its registry entry, or None. - - Exact matches take precedence over `prefix=True` entries, which take - precedence over `dynamic=True` entries matched by `kind`. - - `spans` defaults to Datasette's own registry. - """ - if spans is None: - spans = SPANS - for span in spans: - if span.dynamic: - continue - if emitted_name == span: - return span - for span in spans: - if span.prefix and emitted_name.startswith(span): - return span - if kind is not None: - for span in spans: - if span.dynamic and span.kind == kind: - return span - return None - - -def metric_for(emitted_name, metrics=None): - """ - Resolve an emitted metric name to its registry entry, or None. - - `metrics` defaults to Datasette's own registry. - """ - if metrics is None: - metrics = METRICS - for metric in metrics: - if emitted_name == metric: - return metric - return None - - -def attribute_allowed(entry, emitted_key): - """ - Whether `emitted_key` is a registered attribute of `entry`. - - `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`. - """ - if entry is None: - return False - return emitted_key in entry.attributes - - -def attribute_value_allowed(entry, emitted_key, value): - """ - Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName` - or a `MetricName`). - - Any value is allowed if the attribute does not declare `values=`. - """ - if entry is None: - return False - for attribute in entry.attributes: - if attribute == emitted_key: - return attribute.values is None or value in attribute.values - return False - - -# --- Metrics -------------------------------------------------------------- - -# Bucket boundaries in seconds for every duration histogram. OpenTelemetry's -# defaults are designed for milliseconds and would put almost every SQLite -# query in the first bucket. These are the semantic conventions' recommended -# boundaries for db.client.operation.duration, plus 0.0001 and 0.0005 for -# fast in-process SQLite queries. -DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10) - -M_OPERATION_DURATION = MetricName( - "db.client.operation.duration", - HISTOGRAM, - "s", - "Duration of a SQL operation, including callback-based calls such as " - "``execute_fn()``. For ``block=False`` writes, measures enqueue time.", - (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE), - buckets=DURATION_BUCKETS, -) - -M_WRITE_QUEUE_WAIT = MetricName( - "datasette.write.queue_wait", - HISTOGRAM, - "s", - "Time each write waited in its database's write queue.", - (DB_NAMESPACE,), - buckets=DURATION_BUCKETS, -) - -M_QUERIES_INTERRUPTED = MetricName( - "datasette.sql.queries.interrupted", - COUNTER, - "{query}", - "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A " - "rising rate can indicate that queries need optimization or a higher " - "time limit. Caller-selected timeouts shorter than this limit, such as " - "those used for facet suggestion, are excluded.", - (DB_NAMESPACE,), -) - -M_THREADS_LIMIT = MetricName( - "datasette.sql.threads.limit", - GAUGE, - "{thread}", - "Maximum concurrent read queries, configured by " - ":ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` " - "is ``0``.", -) - -M_THREADS_QUEUE_DEPTH = MetricName( - "datasette.sql.threads.queue_depth", - GAUGE, - "{query}", - "Read queries waiting for a free SQL thread. Sustained values above " - "zero indicate a saturated read pool.", -) - -M_QUERIES_PENDING = MetricName( - "datasette.sql.queries.pending", - GAUGE, - "{query}", - "Read queries submitted to the pool and not yet complete. Sum across " - "databases and compare with ``datasette.sql.threads.limit`` to assess " - "pool usage.", - (DB_NAMESPACE,), -) - -M_WRITE_QUEUE_DEPTH = MetricName( - "datasette.write.queue_depth", - GAUGE, - "{write}", - "Writes waiting for a database's single write thread. Increasing " - "``num_sql_threads`` does not increase write concurrency. Not reported for " - "databases that have never been written to.", - (DB_NAMESPACE,), -) - -M_CONNECTIONS_OPEN = MetricName( - "datasette.connections.open", - GAUGE, - "{connection}", - "Open SQLite connections managed by Datasette.", - (DB_NAMESPACE,), -) - -METRICS = ( - M_OPERATION_DURATION, - M_WRITE_QUEUE_WAIT, - M_QUERIES_INTERRUPTED, - M_THREADS_LIMIT, - M_THREADS_QUEUE_DEPTH, - M_QUERIES_PENDING, - M_WRITE_QUEUE_DEPTH, - M_CONNECTIONS_OPEN, -) diff --git a/datasette/telemetry_testing.py b/datasette/telemetry_testing.py deleted file mode 100644 index 77a4431b..00000000 --- a/datasette/telemetry_testing.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -Pytest helpers for testing OpenTelemetry instrumentation - Datasette's own -and any plugin's. Part of Datasette's public plugin API; see the "Telemetry -for plugin authors" documentation. - -Usage from a plugin's ``conftest.py``:: - - from datasette.telemetry_testing import ( # noqa: F401 - MetricsCollector, - otel_metrics, - otel_meter_provider, - otel_provider, - otel_spans, - ) - -Tests can then use the ``otel_spans`` and ``otel_metrics`` fixtures. The -OpenTelemetry SDK is imported lazily, and the fixtures skip if it is not -installed. -""" - -import subprocess -import sys - -import pytest - -from .telemetry_registry import ( - attribute_allowed, - attribute_value_allowed, - metric_for, - span_for, -) - -_span_exporter = None -_metric_reader = None - - -def install_span_exporter(): - """ - Install a TracerProvider + InMemorySpanExporter once per process and - return the exporter, or None when the SDK is not installed. - - Uses `SimpleSpanProcessor` so spans are exported as soon as they end. - """ - global _span_exporter - if _span_exporter is not None: - return _span_exporter - try: - from opentelemetry import trace as otel_trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, - ) - except ImportError: - return None - exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(exporter)) - otel_trace.set_tracer_provider(provider) - # set_tracer_provider() is ignored if a provider was already installed, - # in which case the fixtures skip - if otel_trace.get_tracer_provider() is not provider: - return None - _span_exporter = exporter - return exporter - - -def install_metric_reader(): - """ - Install a MeterProvider + InMemoryMetricReader once per process and - return the reader, or None when the SDK is not installed. - - Uses delta temporality for counters and histograms, so each collection - only reports measurements since the previous one. - """ - global _metric_reader - if _metric_reader is not None: - return _metric_reader - try: - from opentelemetry import metrics as otel_metrics_api - from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider - from opentelemetry.sdk.metrics.export import ( - AggregationTemporality, - InMemoryMetricReader, - ) - except ImportError: - return None - reader = InMemoryMetricReader( - preferred_temporality={ - Counter: AggregationTemporality.DELTA, - Histogram: AggregationTemporality.DELTA, - } - ) - provider = MeterProvider(metric_readers=[reader]) - otel_metrics_api.set_meter_provider(provider) - if otel_metrics_api.get_meter_provider() is not provider: - return None - _metric_reader = reader - return reader - - -@pytest.fixture(scope="session", autouse=True) -def otel_provider(): - "Install the span exporter once per test session, before any spans are created." - install_span_exporter() - - -@pytest.fixture(scope="session", autouse=True) -def otel_meter_provider(): - "Install the metric reader once per test session." - install_metric_reader() - - -@pytest.fixture(autouse=True) -def otel_reset(): - "Clear recorded spans and drain collected metrics after every test." - yield - if _span_exporter is not None: - _span_exporter.clear() - if _metric_reader is not None: - _metric_reader.get_metrics_data() - - -@pytest.fixture -def otel_spans(): - """ - The in-memory span exporter, cleared before the test. Call - `.get_finished_spans()` to retrieve spans. - """ - pytest.importorskip("opentelemetry.sdk") - exporter = install_span_exporter() - if exporter is None: - pytest.skip("OpenTelemetry SDK provider was not installed") - exporter.clear() - yield exporter - - -class MetricsCollector: - """ - Wraps an `InMemoryMetricReader`. - - `collect()` runs a collection cycle and stores a snapshot, which - `points()` and `point()` then query. - """ - - def __init__(self, reader): - self.reader = reader - self.snapshot = {} - # (instrumentation scope name, sdk Metric) pairs from the last collect() - self.collected = [] - - def collect(self): - self.snapshot = {} - self.collected = [] - data = self.reader.get_metrics_data() - if data is None: - return self.snapshot - for resource_metrics in data.resource_metrics: - for scope_metrics in resource_metrics.scope_metrics: - scope_name = scope_metrics.scope.name if scope_metrics.scope else None - for metric in scope_metrics.metrics: - self.snapshot.setdefault(metric.name, []).extend( - metric.data.data_points - ) - self.collected.append((scope_name, metric)) - return self.snapshot - - def points(self, name, attributes=None): - "Data points for `name` whose attributes are a superset of `attributes`." - found = [] - for point in self.snapshot.get(name, []): - point_attributes = dict(point.attributes or {}) - if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()): - found.append(point) - return found - - def point(self, name, attributes=None): - "The single matching data point, asserting there is exactly one." - found = self.points(name, attributes) - assert len(found) == 1, ( - f"expected exactly one {name} point matching {attributes}, " - f"got {len(found)}: {found}" - ) - return found[0] - - -@pytest.fixture -def otel_metrics(): - "A `MetricsCollector`, drained before the test so counts start from zero." - pytest.importorskip("opentelemetry.sdk") - reader = install_metric_reader() - if reader is None: - pytest.skip("OpenTelemetry SDK meter provider was not installed") - reader.get_metrics_data() - yield MetricsCollector(reader) - - -def _scoped(finished_spans, scope_name): - if scope_name is None: - return list(finished_spans) - return [ - span - for span in finished_spans - if span.instrumentation_scope and span.instrumentation_scope.name == scope_name - ] - - -def assert_spans_conform(registry_spans, finished_spans, scope_name=None): - """ - Assert every finished span is registered in `registry_spans`, sets only - registered attributes and uses allowed attribute values. - - Pass `scope_name` to only check spans from that instrumentation scope. - """ - problems = [] - for span in _scoped(finished_spans, scope_name): - entry = span_for(str(span.name), kind=span.kind, spans=registry_spans) - if entry is None: - problems.append(f"unregistered span: {span.name!r}") - continue - for key, value in (span.attributes or {}).items(): - if not attribute_allowed(entry, str(key)): - problems.append(f"{span.name}: unregistered attribute {key!r}") - elif not attribute_value_allowed(entry, str(key), value): - problems.append( - f"{span.name}: {key}={value!r} not in the declared enum" - ) - assert not problems, "\n".join(problems) - - -def assert_spans_covered(registry_spans, finished_spans, scope_name=None): - """ - Assert every entry in `registry_spans` was emitted at least once, with - each of its attributes that is not `optional=True`. - """ - spans = _scoped(finished_spans, scope_name) - seen_attributes = {} - for span in spans: - entry = span_for(str(span.name), kind=span.kind, spans=registry_spans) - if entry is not None: - seen = seen_attributes.setdefault(str(entry), set()) - seen.update(str(key) for key in (span.attributes or {})) - problems = [] - for entry in registry_spans: - if str(entry) not in seen_attributes: - problems.append(f"registered span never emitted: {entry!r}") - continue - required = { - str(attribute) for attribute in entry.attributes if not attribute.optional - } - missing = required - seen_attributes[str(entry)] - if missing: - problems.append( - f"{entry}: registered attributes never emitted: {sorted(missing)}" - ) - assert not problems, "\n".join(problems) - - -# Registry instrument kinds mapped to the SDK data type collected for them. -# Both counter kinds collect as Sum, distinguished by is_monotonic. -_KIND_TO_DATA_TYPE = { - "Counter": "Sum", - "UpDownCounter": "Sum", - "Histogram": "Histogram", - "Observable gauge": "Gauge", -} -_KIND_IS_MONOTONIC = {"Counter": True, "UpDownCounter": False} - - -def _scoped_metrics(collector, scope_name): - for scope, metric in collector.collected: - if scope_name is None or scope == scope_name: - yield metric - - -def assert_metrics_conform(registry_metrics, collector, scope_name=None): - """ - Assert every metric in the collector's last `collect()` is registered in - `registry_metrics` with a matching instrument kind and unit, sets only - registered attributes and uses allowed attribute values. - - Pass `scope_name` to only check metrics from that instrumentation scope. - """ - problems = set() - for metric in _scoped_metrics(collector, scope_name): - entry = metric_for(metric.name, metrics=registry_metrics) - if entry is None: - problems.add(f"unregistered metric: {metric.name!r}") - continue - expected_data_type = _KIND_TO_DATA_TYPE.get(entry.kind) - actual_data_type = type(metric.data).__name__ - if expected_data_type is not None and actual_data_type != expected_data_type: - problems.add( - f"{metric.name}: registry declares {entry.kind}, " - f"SDK collected {actual_data_type}" - ) - expected_monotonic = _KIND_IS_MONOTONIC.get(entry.kind) - actual_monotonic = getattr(metric.data, "is_monotonic", None) - if ( - expected_monotonic is not None - and actual_monotonic is not None - and actual_monotonic != expected_monotonic - ): - problems.add( - f"{metric.name}: registry declares {entry.kind}, but the " - f"collected Sum is_monotonic={actual_monotonic}" - ) - if (metric.unit or "") != (entry.unit or ""): - problems.add( - f"{metric.name}: instrument unit {metric.unit!r} != " - f"registry unit {entry.unit!r}" - ) - for point in metric.data.data_points: - for key, value in dict(point.attributes or {}).items(): - if not attribute_allowed(entry, str(key)): - problems.add(f"{metric.name}: unregistered attribute {key!r}") - elif not attribute_value_allowed(entry, str(key), value): - problems.add( - f"{metric.name}: {key}={value!r} not in the declared enum" - ) - assert not problems, "\n".join(sorted(problems)) - - -def assert_metrics_covered(registry_metrics, collector, scope_name=None): - """ - Assert every entry in `registry_metrics` was collected at least once, - with each of its attributes that is not `optional=True`. - - Call `collect()` once after the workload and before this check. - """ - seen_attributes = {} - for metric in _scoped_metrics(collector, scope_name): - entry = metric_for(metric.name, metrics=registry_metrics) - if entry is None: - continue - seen = seen_attributes.setdefault(str(entry), set()) - for point in metric.data.data_points: - seen.update(str(key) for key in dict(point.attributes or {})) - problems = [] - for entry in registry_metrics: - if str(entry) not in seen_attributes: - problems.append(f"registered metric never collected: {entry!r}") - continue - required = { - str(attribute) for attribute in entry.attributes if not attribute.optional - } - missing = required - seen_attributes[str(entry)] - if missing: - problems.append( - f"{entry}: registered attributes never collected: {sorted(missing)}" - ) - assert not problems, "\n".join(problems) - - -def assert_no_forbidden_values( - forbidden, finished_spans=None, collector=None, scope_name=None -): - """ - Assert that none of the `forbidden` strings appear anywhere in the - emitted telemetry: span names, span attribute values, span event names - and attributes, span status descriptions, or metric point attributes. - - Use fake private values such as tokens or email addresses in your test - workload, then check that they were not recorded: - - FORBIDDEN = {"secret-token-123", "alice@example.com"} - run_workload_using_those_values() - assert_no_forbidden_values( - FORBIDDEN, - finished_spans=otel_spans.get_finished_spans(), - collector=otel_metrics, - ) - - Matches substrings of each value's string form. Empty strings in - `forbidden` are ignored. Leave `scope_name` unset to also check - Datasette's own telemetry. - """ - needles = [needle for needle in forbidden if needle] - leaks = set() - - def check(value, where): - text = str(value) - for needle in needles: - if needle in text: - leaks.add(f"{where} contains {needle!r}") - - if finished_spans is not None: - for span in _scoped(finished_spans, scope_name): - check(span.name, f"span name {str(span.name)!r}") - for key, value in (span.attributes or {}).items(): - check(value, f"{span.name} attribute {key}") - for event in span.events or (): - check(event.name, f"{span.name} event name") - for key, value in (event.attributes or {}).items(): - check(value, f"{span.name} event {event.name} attribute {key}") - if span.status is not None and span.status.description: - check(span.status.description, f"{span.name} status description") - if collector is not None: - for metric in _scoped_metrics(collector, scope_name): - for point in metric.data.data_points: - for key, value in dict(point.attributes or {}).items(): - check(value, f"metric {metric.name} attribute {key}") - assert not leaks, "forbidden values leaked into telemetry:\n" + "\n".join( - sorted(leaks) - ) - - -def assert_package_never_imports_sdk(*module_names): - """ - Import the named modules in a fresh interpreter and assert none of them - imported `opentelemetry.sdk`. - - Run the test that calls this early in your suite: on macOS with CPython - 3.13, starting a subprocess from a process with many threads can crash. - """ - imports = "; ".join(f"import {name}" for name in module_names) - code = ( - f"import sys; {imports}; " - "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])" - ) - result = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True, check=True - ) - assert result.stdout.strip() == "[]", ( - f"importing {module_names} pulled in the OpenTelemetry SDK: " - f"{result.stdout.strip()}" - ) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 32686af1..4927cb8d 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,6 +3,7 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} + {% endblock %} {% block content %} @@ -125,7 +126,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -173,7 +174,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index e5aa46f3..18288439 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -8,7 +8,6 @@ {% endfor %} - {% for url in extra_js_urls %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index c73cdfb7..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,6 +3,7 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -197,7 +198,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } function displayError(data) { @@ -207,7 +208,7 @@ function displayError(data) { resultsContent.innerHTML = `
      Error: ${escapeHtml(data.error || 'Unknown error')}
      `; - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index c0081c66..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,6 +3,7 @@ {% block title %}Explain a permission decision{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %}