diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b7f2361f..8112af24 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,24 +14,46 @@ jobs: deploy: runs-on: ubuntu-latest steps: + - name: Check deployment prerequisites + id: deployment-prerequisites + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} + run: | + missing=() + for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do + if [[ -z "${!variable:-}" ]]; then + missing+=("$variable") + fi + done + if (( ${#missing[@]} )); then + echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi - name: Check out datasette + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Run tests - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -40,13 +62,14 @@ jobs: plugins \ --extra-db-filename extra_database.db - name: Build docs.db - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -58,6 +81,7 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Deploy to docs as well (only for main) - if: ${{ github.ref == 'refs/heads/main' }} + - name: Upload latest documentation database to S3 (only for main) + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - # Deploy docs.db to a different service - datasette publish cloudrun docs.db \ - --branch=$GITHUB_SHA \ - --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1" \ - --service=datasette-docs-latest + # Keep development documentation separate from the stable release database. + s3-credentials put-object datasette-docs latest/docs.db docs.db \ + --content-type application/octet-stream diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 21ed4c12..232a34c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [created] + types: [published] permissions: contents: read @@ -51,6 +51,8 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 + # After the first non-prerelease 1.0 release, disable this job on 0.65.x, + # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -66,26 +68,20 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy stable-docs.datasette.io to Cloud Run + - name: Upload stable documentation database to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - datasette publish cloudrun docs.db \ - --service=datasette-docs-stable + s3-credentials put-object datasette-docs docs.db docs.db \ + --content-type application/octet-stream deploy_docker: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..58287dd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.11-slim-bookworm AS build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/datasette/app.py b/datasette/app.py index 8cee9b74..3251ed47 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -28,7 +28,7 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx +import httpx2 from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, @@ -49,14 +49,8 @@ from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource -from .telemetry import ( - TelemetryMiddleware, - clamp_http_method, - request_span, - tracer, -) -from .telemetry_registry import HTTP_ROUTE, STARTUP from .tokens import TokenInvalid +from .tracer import AsgiTracer from .url_builder import Urls from .utils import ( SPATIALITE_FUNCTIONS, @@ -293,6 +287,11 @@ SETTINGS = ( False, "Allow display of template debug information with ?_context=1", ), + Setting( + "trace_debug", + False, + "Allow display of SQL trace debug information with ?_trace=1", + ), Setting("base_url", "/", "Datasette URLs should use this base path"), ) _HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead" @@ -316,7 +315,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action, parent, child) + return (actor_key, action.name, parent, action.normalize_child(child)) async def favicon(request, send): @@ -779,73 +778,57 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # `datasette serve` calls invoke_startup() before uvicorn starts, so - # on the CLI path every span its children create - the register_* - # hook dispatches, the internal catalog's db.query/db.write spans, - # and the prepare_connection warm-up of the read connections those - # touch - would otherwise be its own orphan root trace: around twenty - # of them on a fresh instance. Bracketing the whole thing gives them - # somewhere to belong. An ASGI-hosted or programmatic deployment - # reaches here instead through AsgiRunOnFirstRequest, in which case - # this span nests under the first request's own span - honest enough, - # since it genuinely is that request's latency. - # A connection warmed lazily later, by a request touching a new - # database for the first time, nests under that request instead: - # this span has already ended by then. - with tracer.start_as_current_span(STARTUP): - # Register event classes - event_classes = [] - for hook in pm.hook.register_events(datasette=self): - extra_classes = await await_me_maybe(hook) - if extra_classes: - event_classes.extend(extra_classes) - self.event_classes = tuple(event_classes) + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): - if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): - raise StartupError(f"Duplicate action name: {action.name}") - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action + # Register actions, but watch out for duplicate name/abbr + action_names = {} + action_abbrs = {} + for hook in pm.hook.register_actions(datasette=self): + if hook: + for action in hook: + if ( + action.name in action_names + and action != action_names[action.name] + ): + raise StartupError(f"Duplicate action name: {action.name}") + if ( + action.abbr + and action.abbr in action_abbrs + and action != action_abbrs[action.abbr] + ): + raise StartupError(f"Duplicate action abbr: {action.abbr}") + action_names[action.name] = action + if action.abbr: + action_abbrs[action.abbr] = action + self.actions[action.name] = action - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError( - f"Duplicate column type name: {ct_cls.name}" - ) - self._column_types[ct_cls.name] = ct_cls + # Register column types (classes, not instances) + self._column_types = {} + for hook in pm.hook.register_column_types(datasette=self): + if hook: + for ct_cls in hook: + if ct_cls.name in self._column_types: + raise StartupError(f"Duplicate column type name: {ct_cls.name}") + self._column_types[ct_cls.name] = ct_cls - for hook in pm.hook.prepare_jinja2_environment( - env=self._jinja_env, datasette=self - ): - await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() - for hook in pm.hook.startup(datasette=self): - await await_me_maybe(hook) - self._startup_invoked = True + for hook in pm.hook.prepare_jinja2_environment( + env=self._jinja_env, datasette=self + ): + await await_me_maybe(hook) + # Ensure internal tables and metadata are populated before startup hooks + await self._refresh_schemas() + await self._save_queries_from_config() + # Load column_types from config into internal DB + await self._apply_column_types_config() + for hook in pm.hook.startup(datasette=self): + await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -1549,15 +1532,28 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: + # Extension loading is only enabled for as long as it takes to + # load the configured extensions. Leaving it enabled would let + # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) - else: - conn.execute("SELECT load_extension(?)", [extension]) + try: + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + if sys.version_info >= (3, 12): + conn.load_extension(path, entrypoint=entrypoint) + else: + # Connection.load_extension() only gained the + # entrypoint argument in Python 3.12 + conn.execute( + "SELECT load_extension(?, ?)", [path, entrypoint] + ) + else: + conn.load_extension(extension) + finally: + conn.enable_load_extension(False) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1750,8 +1746,145 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) + if action == "view-table": + sql, params = await self._apply_derived_table_permissions_to_sql( + sql, + params, + actor=actor, + parent=parent, + include_is_private=include_is_private, + ) return ResourcesSQL(sql, params) + async def _allowed_derived_table_source( + self, database, source, *, actor, dependencies + ): + """Check an immediate source, denying sources that are themselves derived.""" + if any( + TableResource.normalize_child(table) + == TableResource.normalize_child(source) + for table in dependencies + ): + return False + # The source has no dependency in this map. Evaluate its own permission + # and prerequisites without starting another dependency check. + verdicts = await self._allowed_many( + actions=["view-table"], + resource=TableResource(database, source), + actor=actor, + check_derived=False, + ) + return verdicts["view-table"] + + async def _apply_derived_table_permissions_to_sql( + self, + sql, + params, + *, + actor, + parent, + include_is_private, + ): + databases = ( + [(parent, self.databases[parent])] + if parent in self.databases + else ([] if parent is not None else list(self.databases.items())) + ) + dependency_maps = dict( + zip( + (name for name, _ in databases), + await asyncio.gather( + *(db.derived_table_dependencies() for _, db in databases) + ), + ) + ) + dependencies = [ + (database_name, child, source) + for database_name, dependency_map in dependency_maps.items() + for child, source in dependency_map.items() + ] + if not dependencies: + return sql, params + + sources = sorted( + {(database_name, source) for database_name, _, source in dependencies} + ) + actor_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=actor, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + actor_allowed = dict(zip(sources, actor_verdicts)) + + anonymous_allowed = {} + if include_is_private: + anonymous_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=None, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + anonymous_allowed = dict(zip(sources, anonymous_verdicts)) + + wrapped_params = dict(params) + derived_rows = [ + [ + database_name, + child, + int(actor_allowed[(database_name, source)]), + *( + [int(anonymous_allowed[(database_name, source)])] + if include_is_private + else [] + ), + ] + for database_name, child, source in dependencies + ] + derived_param = "_datasette_derived_permissions" + while derived_param in wrapped_params: + derived_param += "_" + wrapped_params[derived_param] = json.dumps(derived_rows) + + derived_columns = "parent, child, source_allowed" + select_columns = "allowed.parent, allowed.child, allowed.reason" + if include_is_private: + derived_columns += ", source_anonymous_allowed" + select_columns += ( + ", CASE WHEN derived.source_anonymous_allowed = 0 " + "THEN 1 ELSE allowed.is_private END AS is_private" + ) + wrapped_sql = f""" +WITH derived_permissions({derived_columns}) AS ( + SELECT + json_extract(value, '$[0]'), + json_extract(value, '$[1]'), + json_extract(value, '$[2]') + {", json_extract(value, '$[3]')" if include_is_private else ""} + FROM json_each(:{derived_param}) +), +allowed AS ( +{sql} +) +SELECT {select_columns} +FROM allowed +LEFT JOIN derived_permissions AS derived + ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE +WHERE COALESCE(derived.source_allowed, 1) = 1 +ORDER BY allowed.parent, allowed.child +""".strip() + return wrapped_sql, wrapped_params + async def allowed_resources( self, action: str, @@ -1954,6 +2087,12 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ + return await self._allowed_many( + actions=actions, resource=resource, actor=actor, check_derived=True + ) + + async def _allowed_many(self, *, actions, resource, actor, check_derived): + """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, @@ -1991,7 +2130,7 @@ class Datasette: to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, name, parent, child) + key = _permission_cache_key(actor, self.actions[name], parent, child) if key in cache: final[name] = cache[key] continue @@ -2007,6 +2146,28 @@ class Datasette: child=child, ) + if ( + check_derived + and "view-table" in to_check + and raw.get("view-table") + and isinstance(resource, TableResource) + and parent in self.databases + ): + dependencies = await self.databases[parent].derived_table_dependencies() + source = next( + ( + source + for table, source in dependencies.items() + if TableResource.normalize_child(table) + == TableResource.normalize_child(child) + ), + None, + ) + if source is not None: + raw["view-table"] = await self._allowed_derived_table_source( + parent, source, actor=actor, dependencies=dependencies + ) + def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2024,7 +2185,9 @@ class Datasette: # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[_permission_cache_key(actor, name, parent, child)] = final[name] + cache[ + _permission_cache_key(actor, self.actions[name], parent, child) + ] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2466,7 +2629,7 @@ class Datasette: ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + (24 * 60 * 60) + expires_at = int(time.time()) + expire_after data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2832,7 +2995,7 @@ class Datasette: This is the single entry point used by both AsgiLifespan (so real deployments finish startup before accepting requests) and AsgiRunOnFirstRequest (the fallback for hosts that never send - lifespan events, e.g. DatasetteClient's httpx.ASGITransport), and + lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and `datasette serve` (cli.py) calls it too. The fast path below checks both `_startup_invoked` and `_setup_db_done` - not just the former - so that a bare `await ds.invoke_startup()` made by a caller ahead of @@ -2861,6 +3024,8 @@ class Datasette: self.close() asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) + if self.setting("trace_debug"): + asgi = AsgiTracer(asgi) asgi = AsgiLifespan( asgi, on_startup=[self._startup_sequence], @@ -2869,12 +3034,6 @@ class Datasette: asgi = AsgiRunOnFirstRequest(asgi, on_startup=[self._startup_sequence]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) - # Outermost, deliberately: plugin asgi_wrapper() middleware, the - # CSRF layer and the first-request startup fallback all run *inside* - # this span, so a span created by an instrumented plugin - or by - # startup work triggered by the first request - parents to the - # request instead of becoming its own orphan root trace. - asgi = TelemetryMiddleware(asgi) return asgi @@ -2912,6 +3071,50 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) + match, view = resolve_routes(self.routes, path) + is_static = view is favicon or getattr(view, "_datasette_static", False) + original_send = send + + async def send(message): + if message["type"] == "http.response.start" and not ( + is_static and message["status"] in (200, 304) + ): + # Decide privacy after rendering, including for streaming responses + # and error handlers. A public primary resource can still include + # private labels, actor navigation, or cookie-dependent content. + headers = list(message.get("headers", [])) + personalized = ( + request.actor is not None + or "cookie" in request.headers + or "authorization" in request.headers + or any(key.lower() == b"set-cookie" for key, _ in headers) + ) + if personalized: + headers = [ + (key, value) + for key, value in headers + if key.lower() != b"cache-control" + ] + headers.append((b"cache-control", b"private, no-store")) + + # Anonymous responses must not be reused for credentialed requests. + # Preserve any additional variation specified by views or plugins. + vary = [ + part.strip() + for key, value in headers + if key.lower() == b"vary" + for part in value.split(b",") + if part.strip() + ] + if b"*" not in vary: + for name in (b"Cookie", b"Authorization"): + if name.lower() not in {part.lower() for part in vary}: + vary.append(name) + headers = [(k, v) for k, v in headers if k.lower() != b"vary"] + headers.append((b"vary", b", ".join(vary))) + message = dict(message, headers=headers) + await original_send(message) + # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -2951,30 +3154,11 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - - match, view = resolve_routes(self.routes, path) + request.scope = scope if match is None: - # No route matched, so the span keeps the bare method name it was - # given at the edge and gets no http.route. That is what semantic - # conventions ask for when the route is unknown. return await self.handle_404(request, send) - # The request span was started at the ASGI edge, before routing, so it - # carries only the method as a name. Now that the route is known, give - # it the `{method} {route}` shape semantic conventions want, and the - # http.route attribute - the low-cardinality counterpart to url.path, - # and so the one to group by. - span = request_span(scope) - if span is not None: - route = match.re.pattern - span.set_attribute(HTTP_ROUTE, route) - # Clamped, for the same reason the middleware clamps it: the method - # is a client-controlled string, and an unclamped one here would - # put attacker-supplied text back into the span name that the - # middleware just kept out of it. - span.update_name(f"{clamp_http_method(request.method)} {route}") - new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) request.scope = new_scope try: @@ -3285,14 +3469,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3339,10 +3523,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx + **kwargs: Additional arguments to pass to httpx2 Returns: - httpx.Response: The response from the request + httpx2.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3351,16 +3535,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/database.py b/datasette/database.py index efb8c56e..d444cbbf 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,45 +1,19 @@ import asyncio import atexit -import contextvars import inspect import os import queue import sys import tempfile import threading -import time import uuid from collections import namedtuple from pathlib import Path import sqlite_utils -from opentelemetry import context as otel_context_api -from opentelemetry.trace import Link, Status, StatusCode, get_current_span from .inspect import inspect_hash -from .telemetry import sql_attribute, sql_operation_name, tracer -from .telemetry_registry import ( - DB_COLLECTION_NAME, - DB_NAMESPACE, - DB_OPERATION_NAME, - DB_QUERY, - DB_QUERY_EXECUTE, - DB_QUERY_TEXT, - DB_SYSTEM, - DB_WRITE_EXECUTE, - DB_WRITE_QUEUE_WAIT, - EXECUTEMANY, - EXECUTESCRIPT, - INTERRUPTED, - ISOLATED_CONNECTION, - PARAM_COUNT, - PARAM_SETS, - ROWS_RETURNED, - SQL_ERROR_SUPPRESSED, - TIME_LIMIT_MS, - TRANSACTION, - TRUNCATED, -) +from .tracer import trace from .utils import ( call_with_supported_arguments, detect_fts, @@ -55,7 +29,7 @@ from .utils import ( table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_hidden_table_names +from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names connections = threading.local() @@ -111,6 +85,7 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None + self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -272,26 +247,30 @@ class Database: return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, transaction=True, + time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def _inner(conn): + def execute_sql(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - operation_name = sql_operation_name(sql) - if operation_name: - span.set_attribute(DB_OPERATION_NAME, operation_name) - if params: - span.set_attribute(PARAM_COUNT, len(params)) + def _inner(conn): + try: + if time_limit_ms is None: + return execute_sql(conn) + with sqlite_timelimit(conn, time_limit_ms): + return execute_sql(conn) + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + raise + + with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn( _inner, block=block, request=request, transaction=transaction ) @@ -303,15 +282,7 @@ class Database: def _inner(conn): return conn.executescript(sql) - # No db.operation.name here, deliberately: executescript() runs - # several semicolon-separated statements, and semantic conventions - # say the attribute should not be extracted from query text that - # can hold more than one operation - see sql_operation_name(). - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - span.set_attribute(EXECUTESCRIPT, True) + with trace("sql", database=self.name, sql=sql.strip(), executescript=True): results = await self.execute_write_fn( _inner, block=block, transaction=False, request=request ) @@ -331,22 +302,13 @@ class Database: return conn.executemany(sql, count_params(params_seq)), count - with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - span.set_attribute(EXECUTEMANY, True) - # A single statement run with many parameter sets, so unlike - # execute_write_script() there is exactly one operation to name. - operation_name = sql_operation_name(sql) - if operation_name: - span.set_attribute(DB_OPERATION_NAME, operation_name) + with trace( + "sql", database=self.name, sql=sql.strip(), executemany=True + ) as kwargs: results, count = await self.execute_write_fn( _inner, block=block, request=request ) - # count is the number of parameter *sets* consumed by - # executemany(), not a row count - executemany returns no rows. - span.set_attribute(PARAM_SETS, count) + kwargs["count"] = count return results async def execute_isolated_fn(self, fn): @@ -372,18 +334,9 @@ class Database: return _run() if not write: # Immutable database - no writes can ever occur, so there is no - # write queue to block; run against a fresh read-only connection. - # A fresh copy_context() is required per submit (not one shared - # copy reused across calls): concurrent execution of the same - # Context raises "RuntimeError: cannot enter context ... already - # entered". This propagates the caller's otel context (e.g. the - # enclosing db.query span) onto the worker thread. - # - # It also propagates every *other* ContextVar - see the note in - # execute_fn() for why that is safe. - ctx = contextvars.copy_context() + # write queue to block; run against a fresh read-only connection return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, ctx.run, _run + self.ds.executor, _run ) # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) @@ -414,6 +367,15 @@ class Database: result = fn(self._write_connection) else: result = fn(self._write_connection) + if not block: + # There is no write thread here, so the write has already + # finished. Hand back the same (task_id, reply_future) shape + # _send_to_write_thread() returns, with the future already + # resolved, so the block=False path below is identical in + # both modes. + reply_future = asyncio.get_running_loop().create_future() + reply_future.set_result(result) + result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -485,27 +447,11 @@ class Database: ) self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + task_id = uuid.uuid4() loop = asyncio.get_running_loop() reply_future = loop.create_future() - # Captured here, on the event loop, at enqueue time: the otel - # Context (carrying the enclosing db.query span, if any) and the - # timestamp used to build the db.write.queue_wait span once this - # task is dequeued on the write thread. `block` travels with the - # task too, because it decides whether that context is this task's - # parent or only a link target - see `_execute_writes`. self._write_queue.put( - WriteTask( - fn, - task_id, - loop, - reply_future, - isolated_connection, - transaction, - otel_context_api.get_current(), - time.time_ns(), - block, - ) + WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) ) if block: return await reply_future @@ -519,16 +465,6 @@ class Database: conn = None try: conn = self.connect(write=True) - # This warm-up runs before any write has ever been queued, so - # there is no captured caller context to attach - and a raw - # threading.Thread does not inherit the context of whoever started - # it. Spans created by plugin hooks here are therefore roots even - # when the write thread is started from inside invoke_startup(): - # its datasette.startup span is current on the event loop but does - # not cross this thread boundary. Read connections differ - they - # warm up inside executor tasks submitted with copy_context(), so - # their prepare_connection spans do nest under whoever triggered - # them. self.ds._prepare_connection(conn, self.name) except Exception as e: # noqa: BLE001 # Stored and re-raised to whoever queues the next write @@ -543,119 +479,40 @@ class Database: # Best-effort close as the write thread exits pass return - # `task.block` decides how this task's spans relate to the - # context captured at enqueue time: - # - # - block=True: the caller genuinely awaits the reply, so - # containment is accurate. Restore that context as current - # (attach below) so db.write.queue_wait/db.write.execute parent - # normally to the request that queued them. The token must be - # detached below in `finally` - a leaked token silently - # poisons this thread's ambient context for every write - # processed after it, and a *wrong*-token detach only logs a - # warning rather than raising, so this pairing is load-bearing - # and easy to get wrong silently. - # - block=False: the caller returned already without awaiting, - # so the enqueueing span may already have closed (and - # exported) before this task's spans even start - parenting to - # it would make a child appear to outlive its already-closed - # parent, which OTel allows but which renders badly in most - # trace UIs. The enqueueing request *caused* this write - # without *containing* it, so nothing is attached here - - # instead each write span is started as its own root (explicit - # empty `context=`, so the write thread's ambient context - # cannot supply a parent either) carrying one `Link` back to - # the enqueueing span's context, built once into - # `write_span_kwargs` and spread into every start_span call - # below. - token = None - write_span_kwargs = {} - if task.block: - token = otel_context_api.attach(task.otel_context) + exception = None + result = None + if conn_exception is not None: + exception = conn_exception + elif task.isolated_connection: + try: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + except Exception as e: # noqa: BLE001 + # Write thread must survive any task failure or the database wedges + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e else: - enqueueing_span_context = get_current_span( - task.otel_context - ).get_span_context() - # No attributes on the link: there is only one kind of link - # here, so naming the relationship would be a constant that - # carries no information a consumer does not already have - # from the link's existence. - links = ( - [Link(enqueueing_span_context)] - if enqueueing_span_context.is_valid - else [] - ) - write_span_kwargs = { - "context": otel_context_api.Context(), - "links": links, - } - try: - exception = None - result = None - # Explicit start_time/end_time rather than a `with` block: - # this span's duration is the time the task actually spent - # waiting in the queue (enqueue -> dequeue), not the near- - # zero time spent constructing/ending the span object here. - tracer.start_span( - DB_WRITE_QUEUE_WAIT, - start_time=task.enqueued_at_ns, - **write_span_kwargs, - ).end(end_time=time.time_ns()) - if conn_exception is not None: - # fn never runs in this branch, so there is nothing to - # wrap in a db.write.execute span. - exception = conn_exception - elif task.isolated_connection: - try: - with tracer.start_as_current_span( - DB_WRITE_EXECUTE, **write_span_kwargs - ) as span: - span.set_attribute( - ISOLATED_CONNECTION, - task.isolated_connection, - ) - span.set_attribute(TRANSACTION, task.transaction) - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - finally: - isolated_connection.close() - try: - self._all_file_connections.remove( - isolated_connection - ) - except ValueError: - # Was probably a memory connection - pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - else: - try: - with tracer.start_as_current_span( - DB_WRITE_EXECUTE, **write_span_kwargs - ) as span: - span.set_attribute( - ISOLATED_CONNECTION, - task.isolated_connection, - ) - span.set_attribute(TRANSACTION, task.transaction) - if task.transaction: - with conn: - conn.execute("BEGIN IMMEDIATE") - result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) - finally: - if token is not None: - otel_context_api.detach(token) + try: + if task.transaction: + with conn: + conn.execute("BEGIN IMMEDIATE") + result = task.fn(conn) + else: + result = task.fn(conn) + except Exception as e: # noqa: BLE001 + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) async def execute_fn(self, fn): self._check_not_closed() @@ -677,28 +534,7 @@ class Database: with self._pending_execute_futures_lock: self._check_not_closed() - # A fresh copy_context() is required per submit (not one shared - # copy reused across calls): concurrent execution of the same - # Context raises "RuntimeError: cannot enter context ... - # already entered". This propagates the caller's otel context - # (e.g. the enclosing db.query span) onto the worker thread. - # - # copy_context() is not selective: it also carries Datasette's own - # ContextVars - _skip_permission_checks and _permission_check_cache - # (datasette/permissions.py) and _in_datasette_client (app.py) - - # into worker threads, where they previously took their defaults. - # That is safe, for two reasons. Nothing reads them on a worker - # thread: the permission code that reads the first two is async and - # only ever runs on the event loop. And Context.run() restores the - # thread's previous context when the callable returns, so a value - # cannot outlive the submit that carried it and reach the next task - # on this shared pool - "skip permission checks" in particular can - # never bleed from one request into another's query. Where a value - # would be read - a plugin calling datasette.in_client() from inside - # an execute_fn callable - seeing the submitting request's value is - # the more accurate answer, not a leak. - ctx = contextvars.copy_context() - future = self.ds.executor.submit(ctx.run, in_thread) + future = self.ds.executor.submit(in_thread) self._pending_execute_futures.add(future) future.add_done_callback(self._remove_pending_execute_future) return await asyncio.wrap_future(future) @@ -711,143 +547,48 @@ class Database: custom_time_limit=None, page_size=None, log_sql_errors=True, - table=None, ): - """Executes sql against db_name in a thread - - `table`, if passed, is recorded as the `db.collection.name` span - attribute. It exists for callers that already know which table the - query targets - the table and row views - and is never derived from - `sql` itself: deriving it would be a parse, and on an instance where - anyone can create a table the resulting value set has no ceiling. - """ + """Executes sql against db_name in a thread""" self._check_not_closed() page_size = page_size or self.ds.page_size - time_limit_ms = self.ds.sql_time_limit_ms - # A caller that hands in a budget shorter than the instance-wide - # sql_time_limit_ms is saying "this may not finish, and that is an - # answer I can use" - and every such caller in core does treat the - # timeout as normal: table_counts() stores None per table, facet - # suggestion moves on to the next column, autocomplete falls back to a - # prefix query. Those timeouts are therefore not span errors. Without - # this, the homepage alone emits one red span per table (it counts - # every table under a 10ms budget) on every single hit. - # - # A query that runs out the instance-wide limit is a different event - - # nobody asked for a short budget, so it stays an error. - timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms - if timeout_expected: - time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): - # This span is created inside the worker thread. Its parent is - # resolved from the ambient otel context, which was propagated - # onto this thread via copy_context() at the executor.submit() - # boundary in execute_fn() (or run_in_executor() for immutable - # databases) - so it parents correctly to the enclosing - # db.query span despite running on a different thread. - # - # Exception handling is explicit rather than left to the context - # manager's flags, which apply to every exception type alike. This - # span needs to tell two apart: an expected timeout is never an - # error, while a genuine SQL failure is one unless the caller - # passed log_sql_errors=False, meaning it was probing and treats - # failure as an expected answer. Without the latter, facet - # suggestion marks two spans per text column as failed on every - # table page; without the former, so does every homepage hit. - with tracer.start_as_current_span( - DB_QUERY_EXECUTE, - record_exception=False, - set_status_on_exception=False, - ) as execute_span: + time_limit_ms = self.ds.sql_time_limit_ms + if custom_time_limit and custom_time_limit < time_limit_ms: + time_limit_ms = custom_time_limit + + with sqlite_timelimit(conn, time_limit_ms): try: - with sqlite_timelimit(conn, time_limit_ms): - try: - cursor = conn.cursor() - cursor.execute(sql, params if params is not None else {}) - max_returned_rows = self.ds.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - if log_sql_errors: - sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" - ) - sys.stderr.flush() - raise - except QueryInterrupted as e: - if not timeout_expected: - execute_span.record_exception(e) - execute_span.set_status(Status(StatusCode.ERROR, str(e))) - raise - except Exception as e: + cursor = conn.cursor() + cursor.execute(sql, params if params is not None else {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) if log_sql_errors: - execute_span.record_exception(e) - execute_span.set_status(Status(StatusCode.ERROR, str(e))) + sys.stderr.write( + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + ) + sys.stderr.flush() raise - if truncate: - return Results(rows, truncated, cursor.description) + if truncate: + return Results(rows, truncated, cursor.description) - else: - return Results(rows, False, cursor.description) + else: + return Results(rows, False, cursor.description) - # Exception handling is explicit rather than left to the context - # manager's defaults, so that callers passing log_sql_errors=False - # can be honoured - see the comment on the generic handler below. - with tracer.start_as_current_span( - DB_QUERY, - kind=DB_QUERY.kind, - record_exception=False, - set_status_on_exception=False, - ) as span: - span.set_attribute(DB_SYSTEM, "sqlite") - span.set_attribute(DB_NAMESPACE, self.name) - span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) - span.set_attribute(TIME_LIMIT_MS, time_limit_ms) - operation_name = sql_operation_name(sql) - if operation_name: - span.set_attribute(DB_OPERATION_NAME, operation_name) - if table: - span.set_attribute(DB_COLLECTION_NAME, table) - if params: - span.set_attribute(PARAM_COUNT, len(params)) - try: - results = await self.execute_fn(sql_operation_in_thread) - except QueryInterrupted as e: - # datasette.interrupted is set either way - it is the - # signal worth having. Only the ERROR status is - # conditional; see the timeout_expected comment above. - span.set_attribute(INTERRUPTED, True) - if not timeout_expected: - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - raise - except Exception as e: - # log_sql_errors=False means the caller is probing and - # treats failure as an expected answer, not an error. - # Facet suggestion is the big one: it runs json_type() - # against every column precisely to find out which ones - # raise, so a table with N text columns would otherwise - # mark N queries per page as failed - burying real errors - # and setting off any alerting based on span status. - if log_sql_errors: - span.record_exception(e) - span.set_status(Status(StatusCode.ERROR, str(e))) - else: - span.set_attribute(SQL_ERROR_SUPPRESSED, True) - raise - span.set_attribute(TRUNCATED, results.truncated) - span.set_attribute(ROWS_RETURNED, len(results.rows)) + with trace("sql", database=self.name, sql=sql.strip(), params=params): + results = await self.execute_fn(sql_operation_in_thread) return results @property @@ -1040,6 +781,17 @@ class Database: return hidden_tables + async def derived_table_dependencies(self): + """Return implementation tables and the tables they derive from.""" + schema_version = (await self.execute("PRAGMA schema_version")).first()[0] + if ( + self._cached_derived_table_dependencies is None + or self._cached_derived_table_dependencies[0] != schema_version + ): + dependencies = await self.execute_fn(sqlite_derived_table_dependencies) + self._cached_derived_table_dependencies = (schema_version, dependencies) + return self._cached_derived_table_dependencies[1] + async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] @@ -1135,28 +887,16 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( - "block", - "enqueued_at_ns", "fn", "isolated_connection", "loop", - "otel_context", "reply_future", "task_id", "transaction", ) def __init__( - self, - fn, - task_id, - loop, - reply_future, - isolated_connection, - transaction, - otel_context, - enqueued_at_ns, - block, + self, fn, task_id, loop, reply_future, isolated_connection, transaction ): self.fn = fn self.task_id = task_id @@ -1164,14 +904,6 @@ class WriteTask: self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction - self.otel_context = otel_context - self.enqueued_at_ns = enqueued_at_ns - # Whether the enqueueing caller awaits the reply future. Decides how - # `_execute_writes` relates this task's spans to `otel_context`: - # parent (block=True) or span-link target (block=False). See the - # comment at the WriteTask construction site in - # `_send_to_write_thread`. - self.block = block def _deliver_write_result(task, result, exception): diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index f90a733e..6def3698 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,6 +6,17 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType +_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) + + +def _normalize_http_url(value): + if not isinstance(value, str): + return None + normalized = value.strip() + if not _HTTP_URL_RE.fullmatch(normalized): + return None + return normalized + class UrlColumnType(ColumnType): name = "url" @@ -15,7 +26,10 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - escaped = markupsafe.escape(value.strip()) + normalized = _normalize_http_url(value) + if normalized is None: + return markupsafe.escape(value.strip()) + escaped = markupsafe.escape(normalized) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -23,7 +37,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if not re.match(r"^https?://\S+$", value.strip()): + if _normalize_http_url(value) is None: return "Invalid URL" return None diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index 4494f07f..a4f5a4de 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -92,6 +92,13 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) + # Resolve identity keys once per action, rather than scanning the + # restriction allowlist for every configured table's allow block. + self.restricted_table_keys = { + (db, self.action_obj.normalize_child(table) if self.action_obj else table) + for db, table in self.restricted_tables + } + def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: @@ -125,8 +132,10 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - table_actions = table_restrictions.get(child, []) - if self.action_checks.intersection(table_actions): + child_key = ( + self.action_obj.normalize_child(child) if self.action_obj else child + ) + if (parent, child_key) in self.restricted_table_keys: return True else: # Parent query should proceed if any child in this database is allowlisted diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index 88e1d274..d30ebd3f 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -185,11 +185,15 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + action_obj = datasette.actions.get(action) + normalize = action_obj.normalize_child if action_obj else lambda name: name + for table_name, table_allowed in ( + restrictions.get("r", {}).get(database, {}).items() + ): + if normalize(table_name) == normalize(table): + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py new file mode 100644 index 00000000..11fd4008 --- /dev/null +++ b/datasette/default_permissions/sqlite_statistics.py @@ -0,0 +1,25 @@ +"""Default table-access policy for SQLite optimizer statistics.""" + +import json + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + + +@hookimpl +def permission_resources_sql(action): + if action != "view-table": + return None + return PermissionSQL( + sql=""" + SELECT database_name AS parent, value AS child, 0 AS allow, + 'SQLite statistics tables are denied by default' AS reason + FROM catalog_databases + CROSS JOIN json_each(:sqlite_statistics_names) + """, + params={ + "sqlite_statistics_names": json.dumps( + ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] + ) + }, + ) diff --git a/datasette/filters.py b/datasette/filters.py index 3cfb36e5..83e51165 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -2,7 +2,7 @@ import json from typing import ClassVar from datasette import hookimpl -from datasette.resources import DatabaseResource +from datasette.resources import DatabaseResource, TableResource from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError @@ -51,13 +51,20 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which fts_table to use + # Figure out which trusted fts_table to use. Query string parameters can + # repeat this mapping (for backwards compatibility), but must not select + # a different table or primary key. table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = request.args.get("_fts_table") - fts_table = fts_table or table_metadata.get("fts_table") + fts_table = table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) + fts_pk = table_metadata.get("fts_pk", "rowid") + requested_fts_table = request.args.get("_fts_table") + requested_fts_pk = request.args.get("_fts_pk") + if (requested_fts_table and requested_fts_table != fts_table) or ( + requested_fts_pk and requested_fts_pk != fts_pk + ): + raise BadRequest("Invalid _fts_table or _fts_pk") search_args = { key: request.args[key] for key in request.args @@ -75,6 +82,11 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=fts_table), + actor=request.actor, + ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -135,6 +147,11 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=through_table), + actor=request.actor, + ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) fk_to_us = next( diff --git a/datasette/permissions.py b/datasette/permissions.py index e03b065c..2d242560 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -3,6 +3,10 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple +_SQLITE_IDENTIFIER_CASE = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) + # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False @@ -49,6 +53,15 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables + case_insensitive_child: bool = False + + @classmethod + def normalize_child(cls, child: str | None) -> str | None: + """Return a comparison key without changing the resource's display name.""" + if cls.case_insensitive_child and child is not None: + # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. + return child.translate(_SQLITE_IDENTIFIER_CASE) + return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -146,6 +159,11 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed + def normalize_child(self, child: str | None) -> str | None: + if self.resource_class is None: + return child + return self.resource_class.normalize_child(child) + @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 9cf94079..6a4d7da7 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -18,6 +18,7 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", + "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", diff --git a/datasette/resources.py b/datasette/resources.py index ee2e6d98..29bf7b1e 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,6 +25,7 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource + case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index 198641f3..c3d5796c 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -472,11 +472,13 @@ class ColumnChooser extends HTMLElement { - ${col} +
`; + li.querySelector(".drag-item-label").textContent = col; + li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js deleted file mode 100644 index 0e6e2c29..00000000 --- a/datasette/static/json-format-highlight-1.0.1.js +++ /dev/null @@ -1,56 +0,0 @@ -/* -https://github.com/luyilin/json-format-highlight -From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js -MIT Licensed -*/ -(function (global, factory) { - typeof exports === "object" && typeof module !== "undefined" - ? (module.exports = factory()) - : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); -})(this, function () { - "use strict"; - - var defaultColors = { - keyColor: "dimgray", - numberColor: "lightskyblue", - stringColor: "lightcoral", - trueColor: "lightseagreen", - falseColor: "#f66578", - nullColor: "cornflowerblue", - }; - - function index(json, colorOptions) { - if (colorOptions === void 0) colorOptions = {}; - - if (!json) { - return; - } - if (typeof json !== "string") { - json = JSON.stringify(json, null, 2); - } - var colors = Object.assign({}, defaultColors, colorOptions); - json = json.replace(/&/g, "&").replace(//g, ">"); - return json.replace( - /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, - function (match) { - var color = colors.numberColor; - if (/^"/.test(match)) { - color = /:$/.test(match) ? colors.keyColor : colors.stringColor; - } else { - color = /true/.test(match) - ? colors.trueColor - : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; - } - return '' + match + ""; - }, - ); - } - - return index; -}); diff --git a/datasette/telemetry.py b/datasette/telemetry.py deleted file mode 100644 index 311d661f..00000000 --- a/datasette/telemetry.py +++ /dev/null @@ -1,347 +0,0 @@ -""" -OpenTelemetry integration for Datasette core. - -Core depends on `opentelemetry-api` only. It never creates a -`TracerProvider`, never configures an exporter, and never touches -sampling - that is the responsibility of whoever is running Datasette -(an `opentelemetry-instrument` agent, a future plugin, or a test -harness). - -With no provider installed every span produced here is a -`NonRecordingSpan`. That is not free - a table page emits ~58 spans - -but it is below what an end-to-end page benchmark can resolve: measured -across 15 runs of a 5,000-row table page, the median moved 9.80ms to -9.98ms while run-to-run spread was 1.4ms. Installing an SDK provider is -what costs something measurable. -""" - -import re - -from opentelemetry import trace as otel_trace -from opentelemetry.propagate import extract -from opentelemetry.propagators.textmap import Getter -from opentelemetry.trace import SpanKind, Status, StatusCode - -from .telemetry_registry import ( - ERROR_TYPE, - HTTP_REQUEST_METHOD, - HTTP_RESPONSE_STATUS_CODE, - SERVER_ADDRESS, - URL_PATH, - URL_SCHEME, - USER_AGENT_ORIGINAL, -) -from .version import __version__ - -# The semantic-convention version whose spellings this instrumentation -# actually emits. Deliberately NOT the latest release. -# -# A schema URL is a machine-readable claim: a consumer doing schema -# translation replays the renames between the declared version and the one -# it wants, so the claim has to name the version whose spellings are on the -# wire. A wrong one makes translation wrong rather than merely uninformative. -# -# Datasette emits `db.system`, which was renamed to `db.system.name` in -# semconv 1.30.0. Everything else it emits (`db.namespace`, `db.query.text`, -# `db.operation.name`, `db.collection.name`) has been current since 1.26.0. -# So 1.29.0 is the highest version at which every name emitted here is the -# current spelling. Everything under `datasette.*` is Datasette's own and -# outside semconv, so it is unaffected either way. -# -# Declaring 1.43.0 would be false about `db.system`, and would actively STOP -# a consumer translating it forward, because it asserts the rename already -# happened. Bump this deliberately, in the same commit as the attribute -# renames it implies - it is a claim about the names, not decoration. -SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0" - -tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL) - -MAX_SQL_LENGTH = 2048 - - -def sql_attribute(sql: str) -> str: - "Truncate SQL text so it is safe to attach to a span as an attribute." - sql = sql.strip() - if len(sql) <= MAX_SQL_LENGTH: - return sql - return sql[:MAX_SQL_LENGTH] + "…[truncated]" - - -# db.operation.name is the leading keyword of a statement matched against a -# fixed allowlist - deliberately not a parse. -# -# This runs against arbitrary user-supplied SQL (the `?sql=` query string, -# canned queries, anything typed into the query editor), and the attribute is -# a candidate dimension on a query-duration metric in a later phase. A metric -# series is keyed by its attribute values, so echoing back an arbitrary first -# token would let one visitor's typo mint a new, permanent series. The -# allowlist bounds that at a fixed, small set regardless of what anyone sends. -DB_OPERATION_ALLOWLIST = frozenset( - { - "SELECT", - "INSERT", - "UPDATE", - "DELETE", - "CREATE", - "DROP", - "ALTER", - "PRAGMA", - "EXPLAIN", - "REPLACE", - "VACUUM", - "ANALYZE", - "WITH", - } -) - -_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)") - - -def sql_operation_name(sql: str) -> str | None: - """ - The statement's leading keyword, if it is one we recognise. - - Returns None - never a guess - for anything not on the allowlist, - including a statement that opens with a comment or with punctuation such - as the "(" of a parenthesised SELECT. - - Known limitation: a statement beginning with a CTE reports `WITH` rather - than the operation inside it, and a substantial share of Datasette's own - reads take that form. Extracting more than the leading keyword means - handling comment stripping, parenthesised `(SELECT ...) UNION` and - compound names like `CREATE TABLE` - each a special case a hand-rolled - matcher would accrete and eventually get wrong. Omitting a name beats - guessing at one. - - Only safe to call with a single statement: `execute_write_script()` runs - several separated by semicolons, and semantic conventions say - `db.operation.name` "SHOULD NOT be extracted from db.query.text, when the - database system supports query text with multiple operations in non-batch - operations" - so that call site does not use this at all rather than - reporting only the first statement's operation. - """ - match = _LEADING_KEYWORD.match(sql) - if not match: - return None - keyword = match.group(1).upper() - if keyword in DB_OPERATION_ALLOWLIST: - return keyword - return None - - -# --- The HTTP request span ------------------------------------------------ - - -class _ScopeHeadersGetter(Getter): - """ - Read W3C trace context out of an ASGI scope's headers. - - `scope["headers"]` is a list of `(bytes, bytes)` pairs, lowercased by the - server per the ASGI spec - but `.lower()` is applied again here because - that is a spec promise about servers, not something this process - controls. Header bytes are latin-1 by RFC 9110. - """ - - def get(self, carrier, key): - wanted = key.lower().encode("latin-1") - values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted] - return values or None - - def keys(self, carrier): - return [k.decode("latin-1") for k, _ in carrier] - - -_HEADERS_GETTER = _ScopeHeadersGetter() - - -# An unclamped method is an unbounded dimension a client controls: anyone can -# send `FOO / HTTP/1.1`. Semantic conventions say map anything unrecognised to -# `_OTHER`. These nine are the methods of RFC 9110 plus PATCH (RFC 5789). -_KNOWN_METHODS = frozenset( - {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"} -) - - -def clamp_http_method(method): - "The request method if it is one we recognise, else ``_OTHER``." - method = (method or "").upper() - return method if method in _KNOWN_METHODS else "_OTHER" - - -def _first_header(headers, name): - "The first value of a header, decoded, or None." - for key, value in headers: - if key.lower() == name: - return value.decode("latin-1") - return None - - -def _url_path(scope): - """ - The request path, with any query string removed. - - `raw_path` is preferred because it is the bytes the client sent, before - percent-decoding - Datasette routes on database and table names that can - contain encoded slashes, which `scope["path"]` has already collapsed. - - The split on "?" is not decoration. The ASGI spec's `raw_path` excludes - the query string, and uvicorn honours that, but the name is used the - other way round elsewhere in this same dependency tree: httpx's - `URL.raw_path` is documented as "raw bytes of both the path and query". - A server that followed that reading would hand us `?sql=...` here, and - Datasette's query strings carry user-supplied SQL, which core never - records. A literal "?" cannot appear unencoded in a path, so the split - costs nothing when the server is well behaved. - """ - raw_path = scope.get("raw_path") - if raw_path: - if isinstance(raw_path, bytes): - raw_path = raw_path.decode("latin-1") - return raw_path.split("?", 1)[0] - return scope.get("path", "") - - -# The request span is handed to `DatasetteRouter.route_path` through the ASGI -# scope rather than through `get_current_span()`, because by the time routing -# happens the current span may well be something else: a plugin -# `asgi_wrapper()` runs *inside* this middleware, and an instrumented one makes -# its own span current for the whole request. Reading the current span there -# would set `http.route` on that plugin's span - and rename it - while leaving -# the actual request span without the one attribute a trace UI groups by. Not -# hypothetical: an ordinary tracing plugin triggers it. -# -# Namespaced per the ASGI spec's rules for extension keys. Absent when the span -# is not recording, which is exactly when the router should skip the work too. -REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span" - - -def request_span(scope): - """ - The recording request span for an ASGI scope, or None. - - Falls back to the current span so that a `DatasetteRouter` running under - some other instrumentation - one that started a SERVER span but of course - knows nothing about this scope key - still gets enriched. - """ - span = scope.get(REQUEST_SPAN_SCOPE_KEY) - if span is None: - span = otel_trace.get_current_span() - # is_recording(), not `get_span_context().is_valid`: with no provider but - # an inbound `traceparent`, the API's NoOpTracer hands back a - # NonRecordingSpan carrying the *remote* context, which is perfectly valid - # and still records nothing. - return span if span.is_recording() else None - - -class TelemetryMiddleware: - """ - One `SpanKind.SERVER` span per HTTP request. - - Mounted outermost in `Datasette.app()`, so every other span raised while - serving a request - database queries, plugin middleware, startup work on - a cold ASGI-hosted deployment - has somewhere to belong instead of - becoming its own root trace. - - Deliberately much smaller than `opentelemetry-instrumentation-asgi`, - which needs several hundred lines of deferred-end machinery for - applications that return before their body is sent. Datasette does not: - `DatasetteRouter.route_path` awaits `response.asgi_send(send)`, and for a - streaming CSV export `AsgiStream.asgi_send` runs the generator inline. - All of it happens inside the single `await self.app(...)` below, so - ending the span in a `finally` covers the response body too. - """ - - def __init__(self, app): - self.app = app - - async def __call__(self, scope, receive, send): - # First, before anything else: `AsgiLifespan` is *inside* this - # middleware, so lifespan startup and shutdown have to pass through - # untouched or the server never starts. Same for websockets. - if scope["type"] != "http": - await self.app(scope, receive, send) - return - headers = scope.get("headers") or [] - # The *global* propagator, deliberately: it leaves the operator in - # control with no Datasette-specific setting - OTEL_PROPAGATORS=none - # disables extraction entirely, OTEL_PROPAGATORS=tracecontext drops - # baggage - and core configuring propagation itself would be the same - # mistake as core configuring sampling. - context = extract(headers, getter=_HEADERS_GETTER) - method = clamp_http_method(scope.get("method", "")) - # The method, not the URL: a span name has to be low cardinality, and - # the method is what is known out here at the edge, before any routing - # has happened. - with tracer.start_as_current_span( - method, context=context, kind=SpanKind.SERVER - ) as span: - if not span.is_recording(): - # No provider installed, or a sampler dropped this trace. - # Everything below would be discarded, so skip building the - # `send` wrapper and let a default install pay almost - # nothing. Note this cannot be `get_span_context().is_valid`: - # with no provider but an inbound `traceparent`, the API's - # NoOpTracer returns a NonRecordingSpan carrying the *remote* - # context, which is perfectly valid and still records nothing. - await self.app(scope, receive, send) - return - span.set_attribute(HTTP_REQUEST_METHOD, method) - span.set_attribute(URL_PATH, _url_path(scope)) - scheme = scope.get("scheme") - if scheme: - span.set_attribute(URL_SCHEME, scheme) - host = _first_header(headers, b"host") - if host: - span.set_attribute(SERVER_ADDRESS, host) - user_agent = _first_header(headers, b"user-agent") - if user_agent: - span.set_attribute(USER_AGENT_ORIGINAL, user_agent) - - # A copy, not a mutation: the scope belongs to the server, and - # every other layer in Datasette extends it the same way. - scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span}) - - # The status cannot be read off a Response object: `asgi_static`, - # the favicon route, `AsgiStream` and `AsgiFileDownload` all call - # `send` directly and never build one. Wrapping `send` is the only - # thing that sees every response, including the 404 and 500 - # handlers. - status_holder = {} - - async def wrapped_send(message): - if ( - message["type"] == "http.response.start" - and "status" not in status_holder - ): - status_holder["status"] = message["status"] - await send(message) - - escaped = False - try: - # Positional (scope, receive, send) throughout this codebase - - # `wrapped_send` is the third argument. `receive` is passed - # through unwrapped. - await self.app(scope, receive, wrapped_send) - except BaseException as exception: - # BaseException, not Exception: `route_path` turns almost - # everything into a 500 itself, but `asyncio.CancelledError` - # on client disconnect is a BaseException its `except - # Exception` deliberately does not catch. - escaped = True - span.set_attribute(ERROR_TYPE, type(exception).__name__) - span.set_status(Status(StatusCode.ERROR, str(exception))) - raise - finally: - status = status_holder.get("status") - if status is not None: - span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status) - # 4xx is NOT an error for a SERVER span per semantic - # conventions - the client made the mistake, not us. - # - # `not escaped` because this block still runs when an - # exception is on its way out, and a response can have - # started before it: the exception's class name is more - # use than the string "500", so it wins. - if status >= 500 and not escaped: - span.set_status(Status(StatusCode.ERROR)) - span.set_attribute(ERROR_TYPE, str(status)) diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py deleted file mode 100644 index 4e750605..00000000 --- a/datasette/telemetry_registry.py +++ /dev/null @@ -1,388 +0,0 @@ -""" -The single source of truth for every span and span attribute that Datasette -core emits. - -Three things read this module, which is the point of it existing: - -1. **The instrumentation itself.** `Attribute` and `SpanName` subclass `str`, - so a registry entry *is* the string OpenTelemetry wants. Call sites pass - `DB_NAMESPACE` where they used to pass `"db.namespace"` - no wrapper API - over the OTel calls, no parallel structure to keep in step, and a typo is - now an `ImportError` instead of a silently misnamed attribute. - -2. **The documentation.** `docs/telemetry_doc.py` renders the span reference - in `docs/internals.rst` from these definitions using cog, and - `cog --check` runs in CI - so the docs cannot drift from the code. - -3. **A conformance test.** `tests/test_telemetry_registry.py` makes real - requests, collects every span and attribute actually emitted, and compares - both directions: emitted-but-unregistered catches instrumentation added - without documentation, registered-but-never-emitted catches documentation - describing something that no longer exists. Neither the type system nor - the generated docs can catch that second case. -""" - -from opentelemetry.trace import SpanKind - - -class Attribute(str): - """ - A span attribute key, carrying its own documentation. - - Subclasses `str` so it can be handed straight to `set_attribute()`. - """ - - __slots__ = ("description", "optional") - - def __new__(cls, name, description, optional=False): - self = super().__new__(cls, name) - self.description = description - self.optional = optional - return self - - def __repr__(self): - return f"Attribute({str(self)!r})" - - -class SpanName(str): - "A span name, carrying its documentation and the attributes it may set." - - __slots__ = ("attributes", "description", "dynamic", "kind", "prefix") - - def __new__( - cls, - name, - description, - attributes=(), - prefix=False, - dynamic=False, - kind=SpanKind.INTERNAL, - ): - self = super().__new__(cls, name) - self.description = description - self.attributes = tuple(attributes) - # True for a span family whose emitted names carry a variable suffix, - # so the conformance test matches by prefix rather than equality. - # Nothing sets it yet. - self.prefix = prefix - # True when the emitted name is composed at runtime and shares no - # fixed prefix with the registry entry - the HTTP request span, whose - # name is the request method followed by the matched route. There is - # no substring of the entry that could be matched against the wire, so - # `span_for()` resolves these by span kind instead, and the entry's own - # string is a template written for a human reading the generated - # reference. - self.dynamic = dynamic - # SpanKind.INTERNAL by default - every span Datasette emits describes - # its own internal work. db.query is the one exception: it is a real - # database call, so semantic conventions (and trace UIs, which key - # their database styling off this) expect SpanKind.CLIENT. - self.kind = kind - return self - - def __repr__(self): - return f"SpanName({str(self)!r})" - - -# --- Attributes ----------------------------------------------------------- -# -# Shared attributes are defined once and referenced by every span that sets -# them, so "which spans carry db.namespace?" is answerable by grep. - -HTTP_REQUEST_METHOD = Attribute( - "http.request.method", - "The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 " - "define. Anything else is reported as ``_OTHER``: the method is a " - "client-controlled string, so echoing it back unbounded would be a " - "cardinality hazard.", -) -HTTP_RESPONSE_STATUS_CODE = Attribute( - "http.response.status_code", - "The status of the response, read from the ASGI ``http.response.start`` " - "message rather than from a :ref:`internals_response` object - several " - "views, including static files, file downloads and streaming CSV, send " - "that message themselves and never build one. Omitted if the connection " - "closed before anything was sent.", - optional=True, -) -HTTP_ROUTE = Attribute( - "http.route", - "The route the request matched, as the compiled regular expression " - "pattern Datasette routes with - for example " - "``/(?P[^\\/\\.]+)/(?P[^\\/\\.]+)(\\.(?P\\w+))?$`` " - "for a table page. It is deliberately the pattern rather than a prettified " - "``/{database}/{table}`` template: the route table is fixed when the app " - "is built, so the pattern is exact, bounded and needs no parsing, whereas " - "the transform into something prettier accretes edge cases. Unlike " - "``url.path`` this is low cardinality, so it is the attribute to group by. " - "Omitted when no route matched - a 404 - which is also when the span name " - "falls back to the bare method.", - optional=True, -) -URL_PATH = Attribute( - "url.path", - "The path portion of the URL. The query string is deliberately **not** " - "recorded, on this or any other span: Datasette puts user-supplied SQL in " - "``?sql=`` and canned query parameters in the query string, so exporting " - "it by default would export exactly the data the rest of this " - "instrumentation is careful with.", -) -URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.") -SERVER_ADDRESS = Attribute( - "server.address", - "The ``Host`` header. Client-controlled, so treat it as untrusted input " - "rather than as the identity of the server.", - optional=True, -) -USER_AGENT_ORIGINAL = Attribute( - "user_agent.original", - "The ``User-Agent`` header, verbatim. Omitted if the client sent none. " - "The client's IP address is deliberately not recorded: core records no " - "identifier that would tie a span to a person.", - optional=True, -) -ERROR_TYPE = Attribute( - "error.type", - "Set when the request failed: the exception class name if one escaped the " - "application, otherwise the status code as a string for a 5xx response. " - "A 4xx does **not** set this and does not set an error status - per " - "semantic conventions a client error is not a server span's failure.", - optional=True, -) - -DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") -DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") -DB_QUERY_TEXT = Attribute( - "db.query.text", - "The SQL, truncated to 2048 characters. Never the parameter values.", -) -DB_OPERATION_NAME = Attribute( - "db.operation.name", - "The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and " - "so on - matched against a small fixed allowlist. Omitted rather than set " - "to an arbitrary value: the allowlist exists because this attribute is a " - "candidate dimension for a query-duration metric in a later phase, and " - "echoing an unrecognised first token from user-supplied SQL would be an " - "unbounded-cardinality hazard. Also omitted for " - "``execute_write_script()``, which runs multiple statements - per " - "semantic conventions, the operation name should not be extracted from " - "query text that can contain more than one operation. Note that a " - "statement beginning with a CTE reports ``WITH``, not the operation " - "inside it - a substantial share of Datasette's own reads take that " - "form. Resolving it further would mean parsing.", - optional=True, -) -DB_COLLECTION_NAME = Attribute( - "db.collection.name", - "The primary table, set only where the view already knows it - the table " - "and row pages. Omitted for arbitrary ``?sql=`` queries, where determining " - "the table would mean parsing the query.", - optional=True, -) - -PARAM_COUNT = Attribute( - "datasette.param_count", - "Number of bound parameters. Recorded instead of the values themselves.", - optional=True, -) -PARAM_SETS = Attribute( - "datasette.param_sets", - "Number of parameter sets consumed by ``execute_write_many()``. Not a row " - "count - ``executemany()`` returns no rows. The parameter values " - "themselves are never recorded: that sequence can hold thousands of rows.", - optional=True, -) -TIME_LIMIT_MS = Attribute( - "datasette.time_limit_ms", - "The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on " - "reads, which are the queries that time limit applies to.", - optional=True, -) -ROWS_RETURNED = Attribute( - "datasette.rows_returned", - "Number of rows a read returned. Set on the read path only, and only when " - "the read succeeded.", - optional=True, -) -TRUNCATED = Attribute( - "datasette.truncated", - "True if the result was cut short by :ref:`setting_max_returned_rows`.", - optional=True, -) -INTERRUPTED = Attribute( - "datasette.interrupted", - "True if the query was cancelled for exceeding the time limit. The span " - "status is also set to ``ERROR``, unless the caller asked for a budget " - "shorter than :ref:`setting_sql_time_limit_ms` - as table counts, facet " - "suggestion and autocomplete all do - in which case running out of time " - "is an expected answer rather than a failure and the status is left " - "unset.", - optional=True, -) -SQL_ERROR_SUPPRESSED = Attribute( - "datasette.sql_error_suppressed", - "True when the query failed but the caller passed ``log_sql_errors=False``, " - "meaning it was probing and treats failure as an expected answer. Facet " - "suggestion does this against every column.", - optional=True, -) -EXECUTESCRIPT = Attribute( - "datasette.executescript", - "True for ``execute_write_script()``, which runs multiple statements.", - optional=True, -) -EXECUTEMANY = Attribute( - "datasette.executemany", - "True for ``execute_write_many()``, which runs one statement against many " - "parameter sets.", - optional=True, -) -ISOLATED_CONNECTION = Attribute( - "datasette.isolated_connection", - "True if the write ran on its own connection rather than the shared write " - "connection.", -) -TRANSACTION = Attribute( - "datasette.transaction", - "False for statements such as ``VACUUM`` that cannot run inside a transaction.", -) - - -# --- Spans ---------------------------------------------------------------- - -HTTP_REQUEST = SpanName( - "{http.request.method} {http.route}", - "One span per HTTP request, created by the outermost layer of the ASGI " - "stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and " - "every database span raised while serving the request all nest inside " - "it. Without it each of those would be its own root trace. The span name " - "is not a fixed string: it is the method followed by the matched route, " - "and just the method for a request that matched no route. The span starts " - "at the ASGI edge, before routing has happened, so it is named for the " - "method there and renamed once the route is known. " - "W3C ``traceparent`` and ``baggage`` headers are extracted using the " - "global propagator, so a request arriving from an already-traced caller " - "continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, " - "and strip those headers at your proxy if your instance is public.", - ( - HTTP_REQUEST_METHOD, - HTTP_ROUTE, - URL_PATH, - URL_SCHEME, - SERVER_ADDRESS, - USER_AGENT_ORIGINAL, - HTTP_RESPONSE_STATUS_CODE, - ERROR_TYPE, - ), - dynamic=True, - kind=SpanKind.SERVER, -) - -DB_QUERY = SpanName( - "db.query", - "A SQL operation issued by Datasette, covering the full round trip " - "including any time spent queued for a thread.", - ( - DB_SYSTEM, - DB_NAMESPACE, - DB_QUERY_TEXT, - DB_OPERATION_NAME, - DB_COLLECTION_NAME, - PARAM_COUNT, - PARAM_SETS, - TIME_LIMIT_MS, - ROWS_RETURNED, - TRUNCATED, - INTERRUPTED, - SQL_ERROR_SUPPRESSED, - EXECUTESCRIPT, - EXECUTEMANY, - ), - kind=SpanKind.CLIENT, -) - -DB_QUERY_EXECUTE = SpanName( - "db.query.execute", - "The read executing inside a SQL worker thread. Child of ``db.query``; the " - "gap between the two is time spent waiting for a thread.", -) - -DB_WRITE_QUEUE_WAIT = SpanName( - "db.write.queue_wait", - "Time a write spent waiting in its database's write queue before the write " - "thread picked it up. Child of ``db.query`` for a ``block=True`` write, " - "where the caller awaits the write and containment is accurate. For a " - "``block=False`` write the caller does not await it - the enqueueing " - "request *caused* the write without *containing* it, and the write's " - "spans can outlive the request's own - so this is a root span instead, " - "carrying an OpenTelemetry link back to the enqueueing span rather than " - "a parent. A link records causation without asserting containment, which " - "is exactly the distinction here.", -) - -DB_WRITE_EXECUTE = SpanName( - "db.write.execute", - "The write executing on the write thread. Child of ``db.query`` for a " - "``block=True`` write; for ``block=False`` a root span with a link back " - "to the enqueueing span instead - see ``db.write.queue_wait`` above.", - (ISOLATED_CONNECTION, TRANSACTION), -) - -STARTUP = SpanName( - "datasette.startup", - "``invoke_startup()`` running: ``register_events``, ``register_actions``, " - "``register_column_types``, ``prepare_jinja2_environment``, internal-database " - "schema catalog refresh (including the ``prepare_connection`` warm-up this " - "triggers for each database touched for the first time), saved queries, " - "column type config and the ``startup`` hook. Runs once per process, before " - "any request exists, so without this span every child it creates would be " - "its own orphan root trace. A connection warmed later - lazily, the first " - "time a *request* touches a new database or thread - nests under that " - "request's own span instead, not under this one, since this span has " - "already ended by then.", -) - -SPANS = ( - HTTP_REQUEST, - DB_QUERY, - DB_QUERY_EXECUTE, - DB_WRITE_QUEUE_WAIT, - DB_WRITE_EXECUTE, - STARTUP, -) - - -def span_for(emitted_name, kind=None): - """ - Resolve an emitted span name to its registry entry, or None. - - Handles the two entry kinds whose emitted names are not knowable in - advance: - - - `prefix=True` - the name carries a variable suffix, matched by prefix. - Phase 1 registers none. - - `dynamic=True` - the name has no fixed part at all, so it is matched on - `kind` instead and the caller has to supply one. Exact and prefix - entries are tried first, so a dynamic entry can never shadow a span - that does have a registered name. - """ - for span in SPANS: - if span.dynamic: - continue - if span.prefix: - if emitted_name.startswith(span): - return span - elif emitted_name == span: - return span - if kind is not None: - for span in SPANS: - if span.dynamic and span.kind == kind: - return span - return None - - -def attribute_allowed(span, emitted_key): - "Whether `emitted_key` is a registered attribute of `span`." - if span is None: - return False - return emitted_key in span.attributes diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 4927cb8d..32686af1 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,7 +3,6 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} - {% endblock %} {% block content %} @@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 80249d9c..c73cdfb7 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,7 +3,6 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -198,7 +197,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } function displayError(data) { @@ -208,7 +207,7 @@ function displayError(data) { resultsContent.innerHTML = `
Error: ${escapeHtml(data.error || 'Unknown error')}
`; - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index b9fc636a..c0081c66 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,7 +3,6 @@ {% block title %}Explain a permission decision{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %}