diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 3fc83438..cf9b25a7 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,24 +14,46 @@ jobs: deploy: runs-on: ubuntu-latest steps: + - name: Check deployment prerequisites + id: deployment-prerequisites + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} + run: | + missing=() + for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do + if [[ -z "${!variable:-}" ]]; then + missing+=("$variable") + fi + done + if (( ${#missing[@]} )); then + echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi - name: Check out datasette + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Run tests - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -39,14 +61,18 @@ jobs: fixtures-metadata.json \ plugins \ --extra-db-filename extra_database.db + # Package the config with the plugins, excluding test-only plugin secrets + # that reference temporary files outside the deployed container. + jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json - name: Build docs.db - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -58,6 +84,7 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Deploy to docs as well (only for main) - if: ${{ github.ref == 'refs/heads/main' }} + - name: Upload latest documentation database to S3 (only for main) + if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - # Deploy docs.db to a different service - datasette publish cloudrun docs.db \ - --branch=$GITHUB_SHA \ - --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1" \ - --service=datasette-docs-latest + # Keep development documentation separate from the stable release database. + s3-credentials put-object datasette-docs latest/docs.db docs.db \ + --content-type application/octet-stream diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f5b8dbf6..85369f6c 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,9 +2,15 @@ name: Playwright on: push: + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index d92ab82b..fa7ec6aa 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -1,6 +1,15 @@ name: Check JavaScript for conformance with Prettier -on: [push] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 21ed4c12..232a34c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [created] + types: [published] permissions: contents: read @@ -51,6 +51,8 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 + # After the first non-prerelease 1.0 release, disable this job on 0.65.x, + # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -66,26 +68,20 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 + python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy stable-docs.datasette.io to Cloud Run + - name: Upload stable documentation database to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - datasette publish cloudrun docs.db \ - --service=datasette-docs-stable + s3-credentials put-object datasette-docs docs.db docs.db \ + --content-type application/octet-stream deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 58635025..aa35338f 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -1,6 +1,15 @@ name: Check spelling in documentation -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml deleted file mode 100644 index e9bd4bab..00000000 --- a/.github/workflows/test-coverage.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Calculate test coverage - -on: - push: - branches: - - main - pull_request: - branches: - - main -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Check out datasette - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install . --group dev - python -m pip install pytest-cov - - name: Run tests - run: |- - ls -lah - cat .coveragerc - pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x - ls -lah - - name: Upload coverage report - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5e81ed82..449855f3 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper on: push: + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 2fdb3a40..700f3cce 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -1,6 +1,15 @@ name: Test SQLite versions -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -12,10 +21,10 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.13"] sqlite-version: [ #"3", # latest version - "3.46", + #"3.46", #"3.45", #"3.27", #"3.26", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 751eedfd..8176a630 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,15 @@ name: Test -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -11,16 +20,20 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + include: + - python-version: "3.14" + coverage: true steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml + check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) @@ -28,12 +41,27 @@ jobs: run: | pip install . --group dev pip freeze + - name: Install pytest-cov + if: ${{ matrix.coverage }} + run: pip install pytest-cov - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" + if [ "${{ matrix.coverage }}" = "true" ]; then + COV="--cov=datasette --cov-config=.coveragerc" + pytest -n auto -m "not serial" $COV --cov-report= + pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term + else + pytest -n auto -m "not serial" + pytest -m "serial" + fi # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh + - name: Upload coverage report + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml - name: Black run: | black --version diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..58287dd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.11-slim-bookworm AS build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/datasette/__init__.py b/datasette/__init__.py index e0022178..982dcc79 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,6 +1,7 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa +from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor # noqa from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa from datasette.utils.asgi import ( # noqa Forbidden, diff --git a/datasette/app.py b/datasette/app.py index c82ea075..e6e3410e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -28,7 +28,7 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx +import httpx2 from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, @@ -42,6 +42,7 @@ from jinja2.exceptions import TemplateNotFound from markupsafe import Markup, escape from . import stored_queries, write_sql +from .background_tasks import BackgroundTask, BackgroundTaskSupervisor from .column_types import SQLiteType from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted @@ -145,6 +146,7 @@ from .views.stored_queries import ( ) from .views.table import ( TableAutocompleteView, + TableCountView, TableDropView, TableFragmentView, TableInsertView, @@ -315,7 +317,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action, parent, child) + return (actor_key, action.name, parent, action.normalize_child(child)) async def favicon(request, send): @@ -422,6 +424,7 @@ class Datasette: default_deny=False, ): self._startup_invoked = False + self._shutdown_invoked = False self._closed = False assert config_dir is None or isinstance( config_dir, Path @@ -453,8 +456,11 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this + self._setup_db_done = False + self._suppress_background_tasks = False try: self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -462,8 +468,10 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() else: raise + self._background_tasks = BackgroundTaskSupervisor(self) self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: @@ -1529,15 +1537,28 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: + # Extension loading is only enabled for as long as it takes to + # load the configured extensions. Leaving it enabled would let + # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) - else: - conn.execute("SELECT load_extension(?)", [extension]) + try: + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + if sys.version_info >= (3, 12): + conn.load_extension(path, entrypoint=entrypoint) + else: + # Connection.load_extension() only gained the + # entrypoint argument in Python 3.12 + conn.execute( + "SELECT load_extension(?, ?)", [path, entrypoint] + ) + else: + conn.load_extension(extension) + finally: + conn.enable_load_extension(False) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1730,8 +1751,145 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) + if action == "view-table": + sql, params = await self._apply_derived_table_permissions_to_sql( + sql, + params, + actor=actor, + parent=parent, + include_is_private=include_is_private, + ) return ResourcesSQL(sql, params) + async def _allowed_derived_table_source( + self, database, source, *, actor, dependencies + ): + """Check an immediate source, denying sources that are themselves derived.""" + if any( + TableResource.normalize_child(table) + == TableResource.normalize_child(source) + for table in dependencies + ): + return False + # The source has no dependency in this map. Evaluate its own permission + # and prerequisites without starting another dependency check. + verdicts = await self._allowed_many( + actions=["view-table"], + resource=TableResource(database, source), + actor=actor, + check_derived=False, + ) + return verdicts["view-table"] + + async def _apply_derived_table_permissions_to_sql( + self, + sql, + params, + *, + actor, + parent, + include_is_private, + ): + databases = ( + [(parent, self.databases[parent])] + if parent in self.databases + else ([] if parent is not None else list(self.databases.items())) + ) + dependency_maps = dict( + zip( + (name for name, _ in databases), + await asyncio.gather( + *(db.derived_table_dependencies() for _, db in databases) + ), + ) + ) + dependencies = [ + (database_name, child, source) + for database_name, dependency_map in dependency_maps.items() + for child, source in dependency_map.items() + ] + if not dependencies: + return sql, params + + sources = sorted( + {(database_name, source) for database_name, _, source in dependencies} + ) + actor_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=actor, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + actor_allowed = dict(zip(sources, actor_verdicts)) + + anonymous_allowed = {} + if include_is_private: + anonymous_verdicts = await asyncio.gather( + *( + self._allowed_derived_table_source( + database_name, + source, + actor=None, + dependencies=dependency_maps[database_name], + ) + for database_name, source in sources + ) + ) + anonymous_allowed = dict(zip(sources, anonymous_verdicts)) + + wrapped_params = dict(params) + derived_rows = [ + [ + database_name, + child, + int(actor_allowed[(database_name, source)]), + *( + [int(anonymous_allowed[(database_name, source)])] + if include_is_private + else [] + ), + ] + for database_name, child, source in dependencies + ] + derived_param = "_datasette_derived_permissions" + while derived_param in wrapped_params: + derived_param += "_" + wrapped_params[derived_param] = json.dumps(derived_rows) + + derived_columns = "parent, child, source_allowed" + select_columns = "allowed.parent, allowed.child, allowed.reason" + if include_is_private: + derived_columns += ", source_anonymous_allowed" + select_columns += ( + ", CASE WHEN derived.source_anonymous_allowed = 0 " + "THEN 1 ELSE allowed.is_private END AS is_private" + ) + wrapped_sql = f""" +WITH derived_permissions({derived_columns}) AS ( + SELECT + json_extract(value, '$[0]'), + json_extract(value, '$[1]'), + json_extract(value, '$[2]') + {", json_extract(value, '$[3]')" if include_is_private else ""} + FROM json_each(:{derived_param}) +), +allowed AS ( +{sql} +) +SELECT {select_columns} +FROM allowed +LEFT JOIN derived_permissions AS derived + ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE +WHERE COALESCE(derived.source_allowed, 1) = 1 +ORDER BY allowed.parent, allowed.child +""".strip() + return wrapped_sql, wrapped_params + async def allowed_resources( self, action: str, @@ -1934,6 +2092,12 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ + return await self._allowed_many( + actions=actions, resource=resource, actor=actor, check_derived=True + ) + + async def _allowed_many(self, *, actions, resource, actor, check_derived): + """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, @@ -1971,7 +2135,7 @@ class Datasette: to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, name, parent, child) + key = _permission_cache_key(actor, self.actions[name], parent, child) if key in cache: final[name] = cache[key] continue @@ -1987,6 +2151,28 @@ class Datasette: child=child, ) + if ( + check_derived + and "view-table" in to_check + and raw.get("view-table") + and isinstance(resource, TableResource) + and parent in self.databases + ): + dependencies = await self.databases[parent].derived_table_dependencies() + source = next( + ( + source + for table, source in dependencies.items() + if TableResource.normalize_child(table) + == TableResource.normalize_child(child) + ), + None, + ) + if source is not None: + raw["view-table"] = await self._allowed_derived_table_source( + parent, source, actor=actor, dependencies=dependencies + ) + def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2004,7 +2190,9 @@ class Datasette: # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[_permission_cache_key(actor, name, parent, child)] = final[name] + cache[ + _permission_cache_key(actor, self.actions[name], parent, child) + ] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2278,6 +2466,21 @@ class Datasette: ) return d + def _tasks(self): + return { + "tasks": [ + { + "name": t.name, + "state": t.state, + "function": t.function, + "started_at": t.started_at, + "exception": repr(t.exception) if t.exception else None, + } + for t in self._background_tasks.tasks() + ], + "launched": self._background_tasks.launched, + } + def _actor(self, request): return {"actor": request.actor} @@ -2446,7 +2649,7 @@ class Datasette: ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + (24 * 60 * 60) + expires_at = int(time.time()) + expire_after data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2566,6 +2769,12 @@ class Datasette: ), r"/-/threads(\.(?Pjson))?$", ) + add_route( + JsonDataView.as_view( + self, "tasks.json", self._tasks, permission="permissions-debug" + ), + r"/-/tasks(\.(?Pjson))?$", + ) add_route( JsonDataView.as_view( self, @@ -2740,6 +2949,10 @@ class Datasette: TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) + add_route( + TableCountView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", + ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -2803,26 +3016,127 @@ class Datasette: raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) + async def _startup_sequence(self): + """Idempotently run the full startup sequence: table counts for + immutable databases, then invoke_startup(). Safe to call more than + once and safe to call concurrently - callers block until whichever + call got there first has finished. + + This is the single entry point used by both AsgiLifespan (so + real deployments finish startup before accepting requests) and + AsgiRunOnFirstRequest (the fallback for hosts that never send + lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and + `datasette serve` (cli.py) calls it too. The fast path below checks + both `_startup_invoked` and `_setup_db_done` - not just the former - + so that a bare `await ds.invoke_startup()` made by a caller ahead of + `_startup_sequence()` (which only sets `_startup_invoked`) can't + make this method skip the immutable-database table-count precompute. + """ + if self._startup_invoked and self._setup_db_done: + return + async with self._startup_lock: + if self._startup_invoked and self._setup_db_done: + return + if not self._setup_db_done: + # First time server starts up, calculate table counts for + # immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + self._setup_db_done = True + await self.invoke_startup() + + def add_background_task(self, func, name=None) -> BackgroundTask: + """Register a piece of supervised background work, typically from + a plugin's ``startup`` hook. + + ``func`` must be a coroutine function taking one positional + argument, the ``Datasette`` instance - core calls ``func(self)``. + Callable any time after ``__init__``: if background tasks haven't + launched yet (the common case - most callers are ``startup`` hooks, + which run before launch), this buffers the registration until they + do; if they've already launched (e.g. called from a request + handler after the server is up), the task starts immediately. + + Returns a :class:`~datasette.background_tasks.BackgroundTask` + handle (``.name``, ``.state``, ``.task``, ``.exception``, + ``.started_at``, ``.function``, ``.cancel()``). + + ``name`` defaults to ``func.__qualname__``; on a name collision a + ``-2``, ``-3``, ... suffix is appended, since names are how + ``/-/tasks`` and log messages identify work. + """ + return self._background_tasks.add(func, name=name) + + async def start_background_tasks(self): + """Run startup (if it hasn't run yet) and launch every registered + background task. + + Public entry point for tests, embedders, and headless CLIs (the + ``datasette-rss``-style ``fetch --due`` shape) that want supervised + background tasks without running a server - equivalent to what + happens automatically via ASGI lifespan / the first-request + fallback in a served deployment. + """ + await self.invoke_startup() + await self._background_tasks.launch_all() + + async def _launch_background_tasks(self): + """Idempotently launch every registered background task. Private: + this is the entry point wired into the lifecycle trigger lists + (the second entry in both ``AsgiLifespan`` and + ``AsgiRunOnFirstRequest``'s ``on_startup``, after + ``_startup_sequence``) - not something plugins or embedders should + call directly; use ``add_background_task`` / + ``start_background_tasks`` instead. + + Positioned after ``_startup_sequence`` in both trigger lists so + launch always happens once every plugin's ``startup`` hook has had + a chance to register work - the ordering guarantee that makes + ``add_background_task`` useful. No-ops when + ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its + one-shot TestClient request flows through the full ASGI stack, + including the first-request fallback, but must never launch + long-lived background work). + """ + if self._suppress_background_tasks: + return + await self._background_tasks.launch_all() + + async def invoke_shutdown(self): + """Run the graceful teardown sequence: plugin ``shutdown`` hooks, + then cancel and drain supervised background tasks, then close + every database. + """ + if self._shutdown_invoked: + return + self._shutdown_invoked = True + for hook in pm.hook.shutdown(datasette=self): + try: + await await_me_maybe(hook) + except Exception: + logging.getLogger("datasette").exception("shutdown hook failed") + await self._background_tasks.cancel_all(grace=5.0) + self.close() + def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() - async def setup_db(): - # First time server starts up, calculate table counts for immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - - async def _close_on_shutdown(): - self.close() - asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) + asgi = AsgiLifespan( + asgi, + on_startup=[self._startup_sequence, self._launch_background_tasks], + on_shutdown=[self.invoke_shutdown], + ) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) + asgi = AsgiRunOnFirstRequest( + asgi, + on_startup=[self._startup_sequence, self._launch_background_tasks], + ) return asgi @@ -2860,6 +3174,50 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) + match, view = resolve_routes(self.routes, path) + is_static = view is favicon or getattr(view, "_datasette_static", False) + original_send = send + + async def send(message): + if message["type"] == "http.response.start" and not ( + is_static and message["status"] in (200, 304) + ): + # Decide privacy after rendering, including for streaming responses + # and error handlers. A public primary resource can still include + # private labels, actor navigation, or cookie-dependent content. + headers = list(message.get("headers", [])) + personalized = ( + request.actor is not None + or "cookie" in request.headers + or "authorization" in request.headers + or any(key.lower() == b"set-cookie" for key, _ in headers) + ) + if personalized: + headers = [ + (key, value) + for key, value in headers + if key.lower() != b"cache-control" + ] + headers.append((b"cache-control", b"private, no-store")) + + # Anonymous responses must not be reused for credentialed requests. + # Preserve any additional variation specified by views or plugins. + vary = [ + part.strip() + for key, value in headers + if key.lower() == b"vary" + for part in value.split(b",") + if part.strip() + ] + if b"*" not in vary: + for name in (b"Cookie", b"Authorization"): + if name.lower() not in {part.lower() for part in vary}: + vary.append(name) + headers = [(k, v) for k, v in headers if k.lower() != b"vary"] + headers.append((b"vary", b", ".join(vary))) + message = dict(message, headers=headers) + await original_send(message) + # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -2899,8 +3257,7 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - - match, view = resolve_routes(self.routes, path) + request.scope = scope if match is None: return await self.handle_404(request, send) @@ -3215,14 +3572,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3269,10 +3626,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx + **kwargs: Additional arguments to pass to httpx2 Returns: - httpx.Response: The response from the request + httpx2.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3281,16 +3638,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/background_tasks.py b/datasette/background_tasks.py new file mode 100644 index 00000000..6b34fd85 --- /dev/null +++ b/datasette/background_tasks.py @@ -0,0 +1,227 @@ +""" +Supervised background-task registration for Datasette core. + +Plugins that need long-lived background work (a polling loop, a queue +consumer, a scheduled job runner) register it with +``datasette.add_background_task(func, name=None)`` - typically from a +``startup`` plugin hook - instead of fire-and-forgetting their own +``asyncio.create_task()``. Core owns: + +- **references**: every launched ``asyncio.Task`` is kept alive on a + :class:`BackgroundTaskSupervisor`, so it can never be silently garbage + collected the way an unreferenced ``create_task()`` call can be; +- **launch timing**: registered work is buffered until + :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to + happen only after *every* plugin's ``startup`` hook has finished - so + a task that depends on another plugin having registered something first + doesn't need ``tryfirst=True`` ordering tricks; +- **crash surfacing**: an unhandled exception in a background task is + logged with its full traceback to the ``datasette.background_tasks`` + logger and recorded on the handle, instead of becoming an "Task + exception was never retrieved" warning nobody sees; +- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels + every task still running and waits (with a grace period) for them to + actually stop. +""" + +from __future__ import annotations + +import asyncio +import datetime +import functools +import logging +from collections.abc import Awaitable, Callable + +logger = logging.getLogger("datasette.background_tasks") + + +def _utcnow_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _function_path(func: Callable) -> str: + """Describe the callable without guessing which plugin registered it.""" + while isinstance(func, functools.partial): + func = func.func + if not hasattr(func, "__qualname__"): + func = type(func).__call__ + return f"{func.__module__}.{func.__qualname__}" + + +class BackgroundTask: + """A handle to a single piece of supervised background work. + + States: ``registered`` (added but not yet launched) -> ``running`` -> + one of ``completed`` (returned cleanly), ``crashed`` (raised an + exception other than ``CancelledError`` - see ``.exception``), or + ``cancelled`` (``.cancel()`` was called, or it was still running at + shutdown). + """ + + def __init__( + self, + name: str, + func: Callable[[object], Awaitable[None]], + ): + self.name = name + self.state = "registered" + self.task: asyncio.Task | None = None + self.exception: BaseException | None = None + self.started_at: str | None = None + self.function = _function_path(func) + self._func = func + self._supervisor: BackgroundTaskSupervisor | None = None + + def cancel(self) -> None: + """Cancel this task. + + If it has already been launched, cancels the underlying + ``asyncio.Task`` - its state becomes ``cancelled`` once the + cancellation is observed (asynchronously, via the task's done + callback). If it has not been launched yet, this is a no-op as + far as asyncio is concerned (there's no task to cancel) but it + deregisters the handle from its supervisor so it never runs. + """ + if self.task is not None: + self.task.cancel() + elif self._supervisor is not None: + self._supervisor._deregister(self) + + def __repr__(self) -> str: + return f"" + + +class BackgroundTaskSupervisor: + """Owns registration and launch of every :class:`BackgroundTask` for a + single ``Datasette`` instance. + + Registration (:meth:`add`) is separate from launch + (:meth:`launch_all`): plugins register work whenever convenient + (typically from a ``startup`` hook, but request handlers can register + dynamic per-job work too), and it either sits buffered until + :meth:`launch_all` runs, or - if :meth:`launch_all` has already run - + starts immediately. + + Strong references to every :class:`BackgroundTask` (and its + ``asyncio.Task``) are kept for the life of the instance, by design - + that's what makes the enrichments-style "fire-and-forget task gets + garbage collected mid-flight" bug impossible here. There is currently + no pruning of completed/crashed/cancelled tasks, so a plugin that + dynamically registers many short-lived tasks over a long process + lifetime (a per-job registration pattern, e.g. one task per queued + job) will grow this list without bound. That's an accepted v1 + trade-off in favour of full introspection (``/-/tasks``); revisit + with a pruning or capping policy if unbounded growth is reported in + practice. + """ + + def __init__(self, datasette): + self._datasette = datasette + self._tasks: list[BackgroundTask] = [] + self._names = set() + self._launched = False + self._lock = asyncio.Lock() + + def add(self, func, name=None) -> BackgroundTask: + base_name = name or getattr(func, "__qualname__", None) or repr(func) + actual_name = self._unique_name(base_name) + handle = BackgroundTask(actual_name, func) + handle._supervisor = self + self._tasks.append(handle) + self._names.add(actual_name) + if self._launched: + self._launch_one(handle) + return handle + + def _unique_name(self, base_name: str) -> str: + if base_name not in self._names: + return base_name + n = 2 + while f"{base_name}-{n}" in self._names: + n += 1 + return f"{base_name}-{n}" + + def _deregister(self, handle: BackgroundTask) -> None: + try: + self._tasks.remove(handle) + except ValueError: + pass + self._names.discard(handle.name) + + def _launch_one(self, handle: BackgroundTask) -> None: + handle.state = "running" + handle.started_at = _utcnow_iso() + handle.task = asyncio.create_task( + handle._func(self._datasette), name=handle.name + ) + handle.task.add_done_callback(functools.partial(_on_task_done, handle)) + + async def launch_all(self) -> None: + """Launch every currently-registered task that hasn't launched + yet. Idempotent and safe to call concurrently: subsequent (or + racing) calls are no-ops once the first has set ``self._launched``. + """ + if self._launched: + return + async with self._lock: + if self._launched: + return + self._launched = True + for handle in list(self._tasks): + if handle.task is None: + self._launch_one(handle) + + async def cancel_all(self, grace: float = 5.0) -> None: + """Cancel every task that isn't already done, then wait up to + ``grace`` seconds for them to actually finish. Stragglers still + running after that are logged by name (but left to finish or not + on their own - this does not forcibly kill them, asyncio has no + mechanism for that). + """ + handles_by_task = { + handle.task: handle for handle in self._tasks if handle.task is not None + } + pending = [task for task in handles_by_task if not task.done()] + for task in pending: + task.cancel() + if not pending: + return + _done, not_done = await asyncio.wait(pending, timeout=grace) + if not_done: + names = sorted(handles_by_task[task].name for task in not_done) + logger.warning( + "%d background task(s) did not finish within the %.1fs grace " + "period after cancellation: %s", + len(names), + grace, + ", ".join(names), + ) + + def tasks(self) -> list[BackgroundTask]: + """Return every registered :class:`BackgroundTask`, launched or + not, in registration order. Used by the ``/-/tasks`` debug + endpoint. + """ + return list(self._tasks) + + @property + def launched(self) -> bool: + """Whether :meth:`launch_all` has run yet - lets ``/-/tasks`` + distinguish "no tasks registered" from "tasks registered but + nothing has armed the launch yet" without reaching for the + private ``_launched`` attribute. + """ + return self._launched + + +def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None: + if task.cancelled(): + handle.state = "cancelled" + return + exc = task.exception() + if exc is not None: + handle.state = "crashed" + handle.exception = exc + logger.error("Background task %r crashed", handle.name, exc_info=exc) + return + handle.state = "completed" diff --git a/datasette/cli.py b/datasette/cli.py index 57db83b6..4363a28e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -663,16 +663,6 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run async soundness checks before startup hooks, since invoke_startup - # now populates internal tables which requires querying each database - run_sync(lambda: check_databases(ds)) - - # Run the "startup" plugin hooks - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -680,6 +670,19 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: + # --get means we don't run Uvicorn at all + run_sync(lambda: check_databases(ds)) + + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + + # --get never launches background tasks: TestClient's request below + # flows through the full ASGI stack, including the + # AsgiRunOnFirstRequest fallback, which would otherwise launch them. + ds._suppress_background_tasks = True + client = TestClient(ds) request_headers = {} if token: @@ -704,34 +707,54 @@ def serve( sys.exit(exit_code) return - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = run_sync(lambda: initial_path_for_datasette(ds)) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - uvicorn.run(ds.app(), **uvicorn_kwargs) + # check_databases, invoke_startup() and the uvicorn server all run on a + # single event loop, so that anything a plugin's "startup" hook schedules + # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is + # still alive when the server starts handling requests. + async def _serve_async(): + # Populate internal catalog tables before invoke_startup + await check_databases(ds) + + # Run the full startup sequence (immutable-database table-count + # precompute + the "startup" plugin hooks) via the same entry point + # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when + # uvicorn's lifespan.startup fires moments later. + try: + await ds._startup_sequence() + except StartupError as e: + raise click.ClickException(e.args[0]) + + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = await initial_path_for_datasette(ds) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) + await server.serve() + + asyncio.run(_serve_async()) @cli.command() diff --git a/datasette/database.py b/datasette/database.py index e162d34e..d444cbbf 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -29,7 +29,7 @@ from .utils import ( table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_hidden_table_names +from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names connections = threading.local() @@ -85,6 +85,7 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None + self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -246,17 +247,29 @@ class Database: return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, transaction=True, + time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def _inner(conn): + def execute_sql(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) + def _inner(conn): + try: + if time_limit_ms is None: + return execute_sql(conn) + with sqlite_timelimit(conn, time_limit_ms): + return execute_sql(conn) + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + raise + with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn( _inner, block=block, request=request, transaction=transaction @@ -354,6 +367,15 @@ class Database: result = fn(self._write_connection) else: result = fn(self._write_connection) + if not block: + # There is no write thread here, so the write has already + # finished. Hand back the same (task_id, reply_future) shape + # _send_to_write_thread() returns, with the future already + # resolved, so the block=False path below is identical in + # both modes. + reply_future = asyncio.get_running_loop().create_future() + reply_future.set_result(result) + result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -425,7 +447,7 @@ class Database: ) self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + task_id = uuid.uuid4() loop = asyncio.get_running_loop() reply_future = loop.create_future() self._write_queue.put( @@ -759,6 +781,17 @@ class Database: return hidden_tables + async def derived_table_dependencies(self): + """Return implementation tables and the tables they derive from.""" + schema_version = (await self.execute("PRAGMA schema_version")).first()[0] + if ( + self._cached_derived_table_dependencies is None + or self._cached_derived_table_dependencies[0] != schema_version + ): + dependencies = await self.execute_fn(sqlite_derived_table_dependencies) + self._cached_derived_table_dependencies = (schema_version, dependencies) + return self._cached_derived_table_dependencies[1] + async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index f90a733e..6def3698 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,6 +6,17 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType +_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) + + +def _normalize_http_url(value): + if not isinstance(value, str): + return None + normalized = value.strip() + if not _HTTP_URL_RE.fullmatch(normalized): + return None + return normalized + class UrlColumnType(ColumnType): name = "url" @@ -15,7 +26,10 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - escaped = markupsafe.escape(value.strip()) + normalized = _normalize_http_url(value) + if normalized is None: + return markupsafe.escape(value.strip()) + escaped = markupsafe.escape(normalized) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -23,7 +37,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if not re.match(r"^https?://\S+$", value.strip()): + if _normalize_http_url(value) is None: return "Invalid URL" return None diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index 4494f07f..a4f5a4de 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -92,6 +92,13 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) + # Resolve identity keys once per action, rather than scanning the + # restriction allowlist for every configured table's allow block. + self.restricted_table_keys = { + (db, self.action_obj.normalize_child(table) if self.action_obj else table) + for db, table in self.restricted_tables + } + def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: @@ -125,8 +132,10 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - table_actions = table_restrictions.get(child, []) - if self.action_checks.intersection(table_actions): + child_key = ( + self.action_obj.normalize_child(child) if self.action_obj else child + ) + if (parent, child_key) in self.restricted_table_keys: return True else: # Parent query should proceed if any child in this database is allowlisted diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index 88e1d274..d30ebd3f 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -185,11 +185,15 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - table_allowed = restrictions.get("r", {}).get(database, {}).get(table) - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + action_obj = datasette.actions.get(action) + normalize = action_obj.normalize_child if action_obj else lambda name: name + for table_name, table_allowed in ( + restrictions.get("r", {}).get(database, {}).items() + ): + if normalize(table_name) == normalize(table): + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py new file mode 100644 index 00000000..11fd4008 --- /dev/null +++ b/datasette/default_permissions/sqlite_statistics.py @@ -0,0 +1,25 @@ +"""Default table-access policy for SQLite optimizer statistics.""" + +import json + +from datasette import hookimpl +from datasette.permissions import PermissionSQL + + +@hookimpl +def permission_resources_sql(action): + if action != "view-table": + return None + return PermissionSQL( + sql=""" + SELECT database_name AS parent, value AS child, 0 AS allow, + 'SQLite statistics tables are denied by default' AS reason + FROM catalog_databases + CROSS JOIN json_each(:sqlite_statistics_names) + """, + params={ + "sqlite_statistics_names": json.dumps( + ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] + ) + }, + ) diff --git a/datasette/filters.py b/datasette/filters.py index 3cfb36e5..83e51165 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -2,7 +2,7 @@ import json from typing import ClassVar from datasette import hookimpl -from datasette.resources import DatabaseResource +from datasette.resources import DatabaseResource, TableResource from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError @@ -51,13 +51,20 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which fts_table to use + # Figure out which trusted fts_table to use. Query string parameters can + # repeat this mapping (for backwards compatibility), but must not select + # a different table or primary key. table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = request.args.get("_fts_table") - fts_table = fts_table or table_metadata.get("fts_table") + fts_table = table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) + fts_pk = table_metadata.get("fts_pk", "rowid") + requested_fts_table = request.args.get("_fts_table") + requested_fts_pk = request.args.get("_fts_pk") + if (requested_fts_table and requested_fts_table != fts_table) or ( + requested_fts_pk and requested_fts_pk != fts_pk + ): + raise BadRequest("Invalid _fts_table or _fts_pk") search_args = { key: request.args[key] for key in request.args @@ -75,6 +82,11 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=fts_table), + actor=request.actor, + ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -135,6 +147,11 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] + await datasette.ensure_permission( + action="view-table", + resource=TableResource(database=database, table=through_table), + actor=request.actor, + ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) fk_to_us = next( diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index f89f2f36..0b807f8c 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -9,6 +9,11 @@ def startup(datasette): """Fires directly after Datasette first starts running""" +@hookspec +def shutdown(datasette): + """Called once when the Datasette server is shutting down""" + + @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" diff --git a/datasette/permissions.py b/datasette/permissions.py index e03b065c..2d242560 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -3,6 +3,10 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple +_SQLITE_IDENTIFIER_CASE = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) + # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False @@ -49,6 +53,15 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables + case_insensitive_child: bool = False + + @classmethod + def normalize_child(cls, child: str | None) -> str | None: + """Return a comparison key without changing the resource's display name.""" + if cls.case_insensitive_child and child is not None: + # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. + return child.translate(_SQLITE_IDENTIFIER_CASE) + return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -146,6 +159,11 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed + def normalize_child(self, child: str | None) -> str | None: + if self.resource_class is None: + return child + return self.resource_class.normalize_child(child) + @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 9cf94079..6a4d7da7 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -18,6 +18,7 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", + "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", diff --git a/datasette/resources.py b/datasette/resources.py index ee2e6d98..29bf7b1e 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,6 +25,7 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource + case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/app.css b/datasette/static/app.css index d101e4b7..234f535c 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -216,6 +216,49 @@ a:active { text-decoration: underline; } +.table-summary .count-all ~ .table-summary-description { + margin-left: 0.5rem; +} + +.table-summary .count-error:not(:empty) { + display: block; + margin-top: 0.25rem; + font-size: 0.875rem; + font-weight: 400; + line-height: 1.5; +} + +button.count-all { + background: none; + border: none; + padding: 3px 0; + margin-left: 0.25rem; + color: #276890; + font-family: inherit; + font-size: 0.8125rem; + font-weight: 400; + line-height: 1.5; + cursor: pointer; +} + +button.count-all:hover, +button.count-all:focus-visible { + text-decoration: underline; +} + +button.count-all:disabled { + color: #596478; + cursor: wait; +} + +@media (pointer: coarse) { + button.count-all { + min-height: 44px; + padding-left: 7px; + padding-right: 7px; + } +} + button.button-as-link { background: none; border: none; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index 198641f3..c3d5796c 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -472,11 +472,13 @@ class ColumnChooser extends HTMLElement { - ${col} +
`; + li.querySelector(".drag-item-label").textContent = col; + li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js deleted file mode 100644 index 0e6e2c29..00000000 --- a/datasette/static/json-format-highlight-1.0.1.js +++ /dev/null @@ -1,56 +0,0 @@ -/* -https://github.com/luyilin/json-format-highlight -From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js -MIT Licensed -*/ -(function (global, factory) { - typeof exports === "object" && typeof module !== "undefined" - ? (module.exports = factory()) - : typeof define === "function" && define.amd - ? define(factory) - : (global.jsonFormatHighlight = factory()); -})(this, function () { - "use strict"; - - var defaultColors = { - keyColor: "dimgray", - numberColor: "lightskyblue", - stringColor: "lightcoral", - trueColor: "lightseagreen", - falseColor: "#f66578", - nullColor: "cornflowerblue", - }; - - function index(json, colorOptions) { - if (colorOptions === void 0) colorOptions = {}; - - if (!json) { - return; - } - if (typeof json !== "string") { - json = JSON.stringify(json, null, 2); - } - var colors = Object.assign({}, defaultColors, colorOptions); - json = json.replace(/&/g, "&").replace(//g, ">"); - return json.replace( - /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, - function (match) { - var color = colors.numberColor; - if (/^"/.test(match)) { - color = /:$/.test(match) ? colors.keyColor : colors.stringColor; - } else { - color = /true/.test(match) - ? colors.trueColor - : /false/.test(match) - ? colors.falseColor - : /null/.test(match) - ? colors.nullColor - : color; - } - return '' + match + ""; - }, - ); - } - - return index; -}); diff --git a/datasette/static/table.js b/datasette/static/table.js index 74a96d8e..143e976f 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -860,10 +860,45 @@ function openColumnChooser() { }); } +function initCountAll() { + var button = document.querySelector(".count-all"); + if (!button) { + return; + } + button.addEventListener("click", async function () { + var count = document.querySelector(".table-count"); + var error = document.querySelector(".count-error"); + button.disabled = true; + button.textContent = "Counting…"; + error.textContent = ""; + try { + var response = await fetch(button.dataset.countUrl + location.search, { + method: "POST", + headers: { + Accept: "application/json", + }, + }); + var data = await response.json(); + if (!response.ok || !data.ok) { + throw new Error((data.errors || ["Count failed"]).join(" ")); + } + count.textContent = + data.count.toLocaleString("en-US") + + (data.count === 1 ? " row" : " rows"); + button.remove(); + } catch (ex) { + error.textContent = ex.message || "Count failed"; + button.disabled = false; + button.textContent = "count all"; + } + }); +} + // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; + initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 4927cb8d..32686af1 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,7 +3,6 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} - {% endblock %} {% block content %} @@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').innerHTML = jsonFormatHighlight(data); + output.querySelector('pre').textContent = JSON.stringify(data, null, 2); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 80249d9c..c73cdfb7 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,7 +3,6 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -198,7 +197,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } function displayError(data) { @@ -208,7 +207,7 @@ function displayError(data) { resultsContent.innerHTML = `
Error: ${escapeHtml(data.error || 'Unknown error')}
`; - document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); + document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index b9fc636a..c0081c66 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,7 +3,6 @@ {% block title %}Explain a permission decision{% endblock %} {% block extra_head %} - {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %}