diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index cf9b25a7..3fc83438 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,46 +14,24 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Check deployment prerequisites - id: deployment-prerequisites - env: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} - run: | - missing=() - for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do - if [[ -z "${!variable:-}" ]]; then - missing+=("$variable") - fi - done - if (( ${#missing[@]} )); then - echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" - echo "available=false" >> "$GITHUB_OUTPUT" - else - echo "available=true" >> "$GITHUB_OUTPUT" - fi - name: Check out datasette - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -61,18 +39,14 @@ jobs: fixtures-metadata.json \ plugins \ --extra-db-filename extra_database.db - # Package the config with the plugins, excluding test-only plugin secrets - # that reference temporary files outside the deployed container. - jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json - name: Build docs.db - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -84,7 +58,6 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Upload latest documentation database to S3 (only for main) - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - name: Deploy to docs as well (only for main) + if: ${{ github.ref == 'refs/heads/main' }} run: |- - # Keep development documentation separate from the stable release database. - s3-credentials put-object datasette-docs latest/docs.db docs.db \ - --content-type application/octet-stream + # Deploy docs.db to a different service + datasette publish cloudrun docs.db \ + --branch=$GITHUB_SHA \ + --version-note=$GITHUB_SHA \ + --extra-options="--setting template_debug 1" \ + --service=datasette-docs-latest diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 85369f6c..f5b8dbf6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,15 +2,9 @@ name: Playwright on: push: - branches: - - main pull_request: workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - permissions: contents: read diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index fa7ec6aa..d92ab82b 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -1,15 +1,6 @@ name: Check JavaScript for conformance with Prettier -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push] permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 232a34c7..21ed4c12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [published] + types: [created] permissions: contents: read @@ -51,8 +51,6 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 - # After the first non-prerelease 1.0 release, disable this job on 0.65.x, - # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -68,20 +66,26 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Upload stable documentation database to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Deploy stable-docs.datasette.io to Cloud Run run: |- - s3-credentials put-object datasette-docs docs.db docs.db \ - --content-type application/octet-stream + gcloud config set run/region us-central1 + gcloud config set project datasette-222320 + datasette publish cloudrun docs.db \ + --service=datasette-docs-stable deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index aa35338f..58635025 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -1,15 +1,6 @@ name: Check spelling in documentation -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 00000000..e9bd4bab --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,40 @@ +name: Calculate test coverage + +on: + push: + branches: + - main + pull_request: + branches: + - main +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out datasette + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . --group dev + python -m pip install pytest-cov + - name: Run tests + run: |- + ls -lah + cat .coveragerc + pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x + ls -lah + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 449855f3..5e81ed82 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -2,15 +2,9 @@ name: Test in Pyodide with shot-scraper on: push: - branches: - - main pull_request: workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - permissions: contents: read diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 700f3cce..2fdb3a40 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -1,15 +1,6 @@ name: Test SQLite versions -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read @@ -21,10 +12,10 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: ["3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] sqlite-version: [ #"3", # latest version - #"3.46", + "3.46", #"3.45", #"3.27", #"3.26", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8176a630..751eedfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,6 @@ name: Test -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read @@ -20,20 +11,16 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] - include: - - python-version: "3.14" - coverage: true + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) @@ -41,27 +28,12 @@ jobs: run: | pip install . --group dev pip freeze - - name: Install pytest-cov - if: ${{ matrix.coverage }} - run: pip install pytest-cov - name: Run tests run: | - if [ "${{ matrix.coverage }}" = "true" ]; then - COV="--cov=datasette --cov-config=.coveragerc" - pytest -n auto -m "not serial" $COV --cov-report= - pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term - else - pytest -n auto -m "not serial" - pytest -m "serial" - fi + pytest -n auto -m "not serial" + pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Upload coverage report - if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - name: Black run: | black --version diff --git a/Dockerfile b/Dockerfile index 58287dd7..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm AS build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/README.md b/README.md index 1f79778f..393e8e5c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/datasette/__init__.py b/datasette/__init__.py index 982dcc79..e0022178 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,7 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor # noqa from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa from datasette.utils.asgi import ( # noqa Forbidden, diff --git a/datasette/app.py b/datasette/app.py index e6e3410e..c82ea075 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -28,7 +28,7 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx2 +import httpx from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, @@ -42,7 +42,6 @@ from jinja2.exceptions import TemplateNotFound from markupsafe import Markup, escape from . import stored_queries, write_sql -from .background_tasks import BackgroundTask, BackgroundTaskSupervisor from .column_types import SQLiteType from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted @@ -146,7 +145,6 @@ from .views.stored_queries import ( ) from .views.table import ( TableAutocompleteView, - TableCountView, TableDropView, TableFragmentView, TableInsertView, @@ -317,7 +315,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action.name, parent, action.normalize_child(child)) + return (actor_key, action, parent, child) async def favicon(request, send): @@ -424,7 +422,6 @@ class Datasette: default_deny=False, ): self._startup_invoked = False - self._shutdown_invoked = False self._closed = False assert config_dir is None or isinstance( config_dir, Path @@ -456,11 +453,8 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this - self._setup_db_done = False - self._suppress_background_tasks = False try: self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -468,10 +462,8 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() else: raise - self._background_tasks = BackgroundTaskSupervisor(self) self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: @@ -1537,28 +1529,15 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: - # Extension loading is only enabled for as long as it takes to - # load the configured extensions. Leaving it enabled would let - # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - try: - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - if sys.version_info >= (3, 12): - conn.load_extension(path, entrypoint=entrypoint) - else: - # Connection.load_extension() only gained the - # entrypoint argument in Python 3.12 - conn.execute( - "SELECT load_extension(?, ?)", [path, entrypoint] - ) - else: - conn.load_extension(extension) - finally: - conn.enable_load_extension(False) + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1751,145 +1730,8 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) - if action == "view-table": - sql, params = await self._apply_derived_table_permissions_to_sql( - sql, - params, - actor=actor, - parent=parent, - include_is_private=include_is_private, - ) return ResourcesSQL(sql, params) - async def _allowed_derived_table_source( - self, database, source, *, actor, dependencies - ): - """Check an immediate source, denying sources that are themselves derived.""" - if any( - TableResource.normalize_child(table) - == TableResource.normalize_child(source) - for table in dependencies - ): - return False - # The source has no dependency in this map. Evaluate its own permission - # and prerequisites without starting another dependency check. - verdicts = await self._allowed_many( - actions=["view-table"], - resource=TableResource(database, source), - actor=actor, - check_derived=False, - ) - return verdicts["view-table"] - - async def _apply_derived_table_permissions_to_sql( - self, - sql, - params, - *, - actor, - parent, - include_is_private, - ): - databases = ( - [(parent, self.databases[parent])] - if parent in self.databases - else ([] if parent is not None else list(self.databases.items())) - ) - dependency_maps = dict( - zip( - (name for name, _ in databases), - await asyncio.gather( - *(db.derived_table_dependencies() for _, db in databases) - ), - ) - ) - dependencies = [ - (database_name, child, source) - for database_name, dependency_map in dependency_maps.items() - for child, source in dependency_map.items() - ] - if not dependencies: - return sql, params - - sources = sorted( - {(database_name, source) for database_name, _, source in dependencies} - ) - actor_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=actor, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - actor_allowed = dict(zip(sources, actor_verdicts)) - - anonymous_allowed = {} - if include_is_private: - anonymous_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=None, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - anonymous_allowed = dict(zip(sources, anonymous_verdicts)) - - wrapped_params = dict(params) - derived_rows = [ - [ - database_name, - child, - int(actor_allowed[(database_name, source)]), - *( - [int(anonymous_allowed[(database_name, source)])] - if include_is_private - else [] - ), - ] - for database_name, child, source in dependencies - ] - derived_param = "_datasette_derived_permissions" - while derived_param in wrapped_params: - derived_param += "_" - wrapped_params[derived_param] = json.dumps(derived_rows) - - derived_columns = "parent, child, source_allowed" - select_columns = "allowed.parent, allowed.child, allowed.reason" - if include_is_private: - derived_columns += ", source_anonymous_allowed" - select_columns += ( - ", CASE WHEN derived.source_anonymous_allowed = 0 " - "THEN 1 ELSE allowed.is_private END AS is_private" - ) - wrapped_sql = f""" -WITH derived_permissions({derived_columns}) AS ( - SELECT - json_extract(value, '$[0]'), - json_extract(value, '$[1]'), - json_extract(value, '$[2]') - {", json_extract(value, '$[3]')" if include_is_private else ""} - FROM json_each(:{derived_param}) -), -allowed AS ( -{sql} -) -SELECT {select_columns} -FROM allowed -LEFT JOIN derived_permissions AS derived - ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE -WHERE COALESCE(derived.source_allowed, 1) = 1 -ORDER BY allowed.parent, allowed.child -""".strip() - return wrapped_sql, wrapped_params - async def allowed_resources( self, action: str, @@ -2092,12 +1934,6 @@ ORDER BY allowed.parent, allowed.child ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ - return await self._allowed_many( - actions=actions, resource=resource, actor=actor, check_derived=True - ) - - async def _allowed_many(self, *, actions, resource, actor, check_derived): - """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, @@ -2135,7 +1971,7 @@ ORDER BY allowed.parent, allowed.child to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, self.actions[name], parent, child) + key = _permission_cache_key(actor, name, parent, child) if key in cache: final[name] = cache[key] continue @@ -2151,28 +1987,6 @@ ORDER BY allowed.parent, allowed.child child=child, ) - if ( - check_derived - and "view-table" in to_check - and raw.get("view-table") - and isinstance(resource, TableResource) - and parent in self.databases - ): - dependencies = await self.databases[parent].derived_table_dependencies() - source = next( - ( - source - for table, source in dependencies.items() - if TableResource.normalize_child(table) - == TableResource.normalize_child(child) - ), - None, - ) - if source is not None: - raw["view-table"] = await self._allowed_derived_table_source( - parent, source, actor=actor, dependencies=dependencies - ) - def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2190,9 +2004,7 @@ ORDER BY allowed.parent, allowed.child # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[ - _permission_cache_key(actor, self.actions[name], parent, child) - ] = final[name] + cache[_permission_cache_key(actor, name, parent, child)] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2466,21 +2278,6 @@ ORDER BY allowed.parent, allowed.child ) return d - def _tasks(self): - return { - "tasks": [ - { - "name": t.name, - "state": t.state, - "function": t.function, - "started_at": t.started_at, - "exception": repr(t.exception) if t.exception else None, - } - for t in self._background_tasks.tasks() - ], - "launched": self._background_tasks.launched, - } - def _actor(self, request): return {"actor": request.actor} @@ -2649,7 +2446,7 @@ ORDER BY allowed.parent, allowed.child ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + expire_after + expires_at = int(time.time()) + (24 * 60 * 60) data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2769,12 +2566,6 @@ ORDER BY allowed.parent, allowed.child ), r"/-/threads(\.(?Pjson))?$", ) - add_route( - JsonDataView.as_view( - self, "tasks.json", self._tasks, permission="permissions-debug" - ), - r"/-/tasks(\.(?Pjson))?$", - ) add_route( JsonDataView.as_view( self, @@ -2949,10 +2740,6 @@ ORDER BY allowed.parent, allowed.child TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) - add_route( - TableCountView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", - ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -3016,127 +2803,26 @@ ORDER BY allowed.parent, allowed.child raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) - async def _startup_sequence(self): - """Idempotently run the full startup sequence: table counts for - immutable databases, then invoke_startup(). Safe to call more than - once and safe to call concurrently - callers block until whichever - call got there first has finished. - - This is the single entry point used by both AsgiLifespan (so - real deployments finish startup before accepting requests) and - AsgiRunOnFirstRequest (the fallback for hosts that never send - lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and - `datasette serve` (cli.py) calls it too. The fast path below checks - both `_startup_invoked` and `_setup_db_done` - not just the former - - so that a bare `await ds.invoke_startup()` made by a caller ahead of - `_startup_sequence()` (which only sets `_startup_invoked`) can't - make this method skip the immutable-database table-count precompute. - """ - if self._startup_invoked and self._setup_db_done: - return - async with self._startup_lock: - if self._startup_invoked and self._setup_db_done: - return - if not self._setup_db_done: - # First time server starts up, calculate table counts for - # immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - self._setup_db_done = True - await self.invoke_startup() - - def add_background_task(self, func, name=None) -> BackgroundTask: - """Register a piece of supervised background work, typically from - a plugin's ``startup`` hook. - - ``func`` must be a coroutine function taking one positional - argument, the ``Datasette`` instance - core calls ``func(self)``. - Callable any time after ``__init__``: if background tasks haven't - launched yet (the common case - most callers are ``startup`` hooks, - which run before launch), this buffers the registration until they - do; if they've already launched (e.g. called from a request - handler after the server is up), the task starts immediately. - - Returns a :class:`~datasette.background_tasks.BackgroundTask` - handle (``.name``, ``.state``, ``.task``, ``.exception``, - ``.started_at``, ``.function``, ``.cancel()``). - - ``name`` defaults to ``func.__qualname__``; on a name collision a - ``-2``, ``-3``, ... suffix is appended, since names are how - ``/-/tasks`` and log messages identify work. - """ - return self._background_tasks.add(func, name=name) - - async def start_background_tasks(self): - """Run startup (if it hasn't run yet) and launch every registered - background task. - - Public entry point for tests, embedders, and headless CLIs (the - ``datasette-rss``-style ``fetch --due`` shape) that want supervised - background tasks without running a server - equivalent to what - happens automatically via ASGI lifespan / the first-request - fallback in a served deployment. - """ - await self.invoke_startup() - await self._background_tasks.launch_all() - - async def _launch_background_tasks(self): - """Idempotently launch every registered background task. Private: - this is the entry point wired into the lifecycle trigger lists - (the second entry in both ``AsgiLifespan`` and - ``AsgiRunOnFirstRequest``'s ``on_startup``, after - ``_startup_sequence``) - not something plugins or embedders should - call directly; use ``add_background_task`` / - ``start_background_tasks`` instead. - - Positioned after ``_startup_sequence`` in both trigger lists so - launch always happens once every plugin's ``startup`` hook has had - a chance to register work - the ordering guarantee that makes - ``add_background_task`` useful. No-ops when - ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its - one-shot TestClient request flows through the full ASGI stack, - including the first-request fallback, but must never launch - long-lived background work). - """ - if self._suppress_background_tasks: - return - await self._background_tasks.launch_all() - - async def invoke_shutdown(self): - """Run the graceful teardown sequence: plugin ``shutdown`` hooks, - then cancel and drain supervised background tasks, then close - every database. - """ - if self._shutdown_invoked: - return - self._shutdown_invoked = True - for hook in pm.hook.shutdown(datasette=self): - try: - await await_me_maybe(hook) - except Exception: - logging.getLogger("datasette").exception("shutdown hook failed") - await self._background_tasks.cancel_all(grace=5.0) - self.close() - def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() + async def setup_db(): + # First time server starts up, calculate table counts for immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + + async def _close_on_shutdown(): + self.close() + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan( - asgi, - on_startup=[self._startup_sequence, self._launch_background_tasks], - on_shutdown=[self.invoke_shutdown], - ) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) - asgi = AsgiRunOnFirstRequest( - asgi, - on_startup=[self._startup_sequence, self._launch_background_tasks], - ) return asgi @@ -3174,50 +2860,6 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) - match, view = resolve_routes(self.routes, path) - is_static = view is favicon or getattr(view, "_datasette_static", False) - original_send = send - - async def send(message): - if message["type"] == "http.response.start" and not ( - is_static and message["status"] in (200, 304) - ): - # Decide privacy after rendering, including for streaming responses - # and error handlers. A public primary resource can still include - # private labels, actor navigation, or cookie-dependent content. - headers = list(message.get("headers", [])) - personalized = ( - request.actor is not None - or "cookie" in request.headers - or "authorization" in request.headers - or any(key.lower() == b"set-cookie" for key, _ in headers) - ) - if personalized: - headers = [ - (key, value) - for key, value in headers - if key.lower() != b"cache-control" - ] - headers.append((b"cache-control", b"private, no-store")) - - # Anonymous responses must not be reused for credentialed requests. - # Preserve any additional variation specified by views or plugins. - vary = [ - part.strip() - for key, value in headers - if key.lower() == b"vary" - for part in value.split(b",") - if part.strip() - ] - if b"*" not in vary: - for name in (b"Cookie", b"Authorization"): - if name.lower() not in {part.lower() for part in vary}: - vary.append(name) - headers = [(k, v) for k, v in headers if k.lower() != b"vary"] - headers.append((b"vary", b", ".join(vary))) - message = dict(message, headers=headers) - await original_send(message) - # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -3257,7 +2899,8 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - request.scope = scope + + match, view = resolve_routes(self.routes, path) if match is None: return await self.handle_404(request, send) @@ -3572,14 +3215,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3626,10 +3269,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx2 + **kwargs: Additional arguments to pass to httpx Returns: - httpx2.Response: The response from the request + httpx.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3638,16 +3281,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/background_tasks.py b/datasette/background_tasks.py deleted file mode 100644 index 6b34fd85..00000000 --- a/datasette/background_tasks.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Supervised background-task registration for Datasette core. - -Plugins that need long-lived background work (a polling loop, a queue -consumer, a scheduled job runner) register it with -``datasette.add_background_task(func, name=None)`` - typically from a -``startup`` plugin hook - instead of fire-and-forgetting their own -``asyncio.create_task()``. Core owns: - -- **references**: every launched ``asyncio.Task`` is kept alive on a - :class:`BackgroundTaskSupervisor`, so it can never be silently garbage - collected the way an unreferenced ``create_task()`` call can be; -- **launch timing**: registered work is buffered until - :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to - happen only after *every* plugin's ``startup`` hook has finished - so - a task that depends on another plugin having registered something first - doesn't need ``tryfirst=True`` ordering tricks; -- **crash surfacing**: an unhandled exception in a background task is - logged with its full traceback to the ``datasette.background_tasks`` - logger and recorded on the handle, instead of becoming an "Task - exception was never retrieved" warning nobody sees; -- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels - every task still running and waits (with a grace period) for them to - actually stop. -""" - -from __future__ import annotations - -import asyncio -import datetime -import functools -import logging -from collections.abc import Awaitable, Callable - -logger = logging.getLogger("datasette.background_tasks") - - -def _utcnow_iso() -> str: - return datetime.datetime.now(datetime.timezone.utc).isoformat() - - -def _function_path(func: Callable) -> str: - """Describe the callable without guessing which plugin registered it.""" - while isinstance(func, functools.partial): - func = func.func - if not hasattr(func, "__qualname__"): - func = type(func).__call__ - return f"{func.__module__}.{func.__qualname__}" - - -class BackgroundTask: - """A handle to a single piece of supervised background work. - - States: ``registered`` (added but not yet launched) -> ``running`` -> - one of ``completed`` (returned cleanly), ``crashed`` (raised an - exception other than ``CancelledError`` - see ``.exception``), or - ``cancelled`` (``.cancel()`` was called, or it was still running at - shutdown). - """ - - def __init__( - self, - name: str, - func: Callable[[object], Awaitable[None]], - ): - self.name = name - self.state = "registered" - self.task: asyncio.Task | None = None - self.exception: BaseException | None = None - self.started_at: str | None = None - self.function = _function_path(func) - self._func = func - self._supervisor: BackgroundTaskSupervisor | None = None - - def cancel(self) -> None: - """Cancel this task. - - If it has already been launched, cancels the underlying - ``asyncio.Task`` - its state becomes ``cancelled`` once the - cancellation is observed (asynchronously, via the task's done - callback). If it has not been launched yet, this is a no-op as - far as asyncio is concerned (there's no task to cancel) but it - deregisters the handle from its supervisor so it never runs. - """ - if self.task is not None: - self.task.cancel() - elif self._supervisor is not None: - self._supervisor._deregister(self) - - def __repr__(self) -> str: - return f"" - - -class BackgroundTaskSupervisor: - """Owns registration and launch of every :class:`BackgroundTask` for a - single ``Datasette`` instance. - - Registration (:meth:`add`) is separate from launch - (:meth:`launch_all`): plugins register work whenever convenient - (typically from a ``startup`` hook, but request handlers can register - dynamic per-job work too), and it either sits buffered until - :meth:`launch_all` runs, or - if :meth:`launch_all` has already run - - starts immediately. - - Strong references to every :class:`BackgroundTask` (and its - ``asyncio.Task``) are kept for the life of the instance, by design - - that's what makes the enrichments-style "fire-and-forget task gets - garbage collected mid-flight" bug impossible here. There is currently - no pruning of completed/crashed/cancelled tasks, so a plugin that - dynamically registers many short-lived tasks over a long process - lifetime (a per-job registration pattern, e.g. one task per queued - job) will grow this list without bound. That's an accepted v1 - trade-off in favour of full introspection (``/-/tasks``); revisit - with a pruning or capping policy if unbounded growth is reported in - practice. - """ - - def __init__(self, datasette): - self._datasette = datasette - self._tasks: list[BackgroundTask] = [] - self._names = set() - self._launched = False - self._lock = asyncio.Lock() - - def add(self, func, name=None) -> BackgroundTask: - base_name = name or getattr(func, "__qualname__", None) or repr(func) - actual_name = self._unique_name(base_name) - handle = BackgroundTask(actual_name, func) - handle._supervisor = self - self._tasks.append(handle) - self._names.add(actual_name) - if self._launched: - self._launch_one(handle) - return handle - - def _unique_name(self, base_name: str) -> str: - if base_name not in self._names: - return base_name - n = 2 - while f"{base_name}-{n}" in self._names: - n += 1 - return f"{base_name}-{n}" - - def _deregister(self, handle: BackgroundTask) -> None: - try: - self._tasks.remove(handle) - except ValueError: - pass - self._names.discard(handle.name) - - def _launch_one(self, handle: BackgroundTask) -> None: - handle.state = "running" - handle.started_at = _utcnow_iso() - handle.task = asyncio.create_task( - handle._func(self._datasette), name=handle.name - ) - handle.task.add_done_callback(functools.partial(_on_task_done, handle)) - - async def launch_all(self) -> None: - """Launch every currently-registered task that hasn't launched - yet. Idempotent and safe to call concurrently: subsequent (or - racing) calls are no-ops once the first has set ``self._launched``. - """ - if self._launched: - return - async with self._lock: - if self._launched: - return - self._launched = True - for handle in list(self._tasks): - if handle.task is None: - self._launch_one(handle) - - async def cancel_all(self, grace: float = 5.0) -> None: - """Cancel every task that isn't already done, then wait up to - ``grace`` seconds for them to actually finish. Stragglers still - running after that are logged by name (but left to finish or not - on their own - this does not forcibly kill them, asyncio has no - mechanism for that). - """ - handles_by_task = { - handle.task: handle for handle in self._tasks if handle.task is not None - } - pending = [task for task in handles_by_task if not task.done()] - for task in pending: - task.cancel() - if not pending: - return - _done, not_done = await asyncio.wait(pending, timeout=grace) - if not_done: - names = sorted(handles_by_task[task].name for task in not_done) - logger.warning( - "%d background task(s) did not finish within the %.1fs grace " - "period after cancellation: %s", - len(names), - grace, - ", ".join(names), - ) - - def tasks(self) -> list[BackgroundTask]: - """Return every registered :class:`BackgroundTask`, launched or - not, in registration order. Used by the ``/-/tasks`` debug - endpoint. - """ - return list(self._tasks) - - @property - def launched(self) -> bool: - """Whether :meth:`launch_all` has run yet - lets ``/-/tasks`` - distinguish "no tasks registered" from "tasks registered but - nothing has armed the launch yet" without reaching for the - private ``_launched`` attribute. - """ - return self._launched - - -def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None: - if task.cancelled(): - handle.state = "cancelled" - return - exc = task.exception() - if exc is not None: - handle.state = "crashed" - handle.exception = exc - logger.error("Background task %r crashed", handle.name, exc_info=exc) - return - handle.state = "completed" diff --git a/datasette/cli.py b/datasette/cli.py index 7887e642..57db83b6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -497,7 +497,6 @@ def uninstall(packages, yes): "--internal", type=click.Path(), help="Path to a persistent Datasette internal SQLite database", - envvar="DATASETTE_INTERNAL", ) def serve( files, @@ -664,6 +663,16 @@ def serve( # Private utility mechanism for writing unit tests return ds + # Run async soundness checks before startup hooks, since invoke_startup + # now populates internal tables which requires querying each database + run_sync(lambda: check_databases(ds)) + + # Run the "startup" plugin hooks + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -671,19 +680,6 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: - # --get means we don't run Uvicorn at all - run_sync(lambda: check_databases(ds)) - - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - - # --get never launches background tasks: TestClient's request below - # flows through the full ASGI stack, including the - # AsgiRunOnFirstRequest fallback, which would otherwise launch them. - ds._suppress_background_tasks = True - client = TestClient(ds) request_headers = {} if token: @@ -708,54 +704,34 @@ def serve( sys.exit(exit_code) return - # check_databases, invoke_startup() and the uvicorn server all run on a - # single event loop, so that anything a plugin's "startup" hook schedules - # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is - # still alive when the server starts handling requests. - async def _serve_async(): - # Populate internal catalog tables before invoke_startup - await check_databases(ds) - - # Run the full startup sequence (immutable-database table-count - # precompute + the "startup" plugin hooks) via the same entry point - # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when - # uvicorn's lifespan.startup fires moments later. - try: - await ds._startup_sequence() - except StartupError as e: - raise click.ClickException(e.args[0]) - - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = await initial_path_for_datasette(ds) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) - await server.serve() - - asyncio.run(_serve_async()) + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = run_sync(lambda: initial_path_for_datasette(ds)) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + uvicorn.run(ds.app(), **uvicorn_kwargs) @cli.command() diff --git a/datasette/database.py b/datasette/database.py index d444cbbf..e162d34e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -29,7 +29,7 @@ from .utils import ( table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names +from .utils.sqlite import sqlite_hidden_table_names connections = threading.local() @@ -85,7 +85,6 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None - self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -247,29 +246,17 @@ class Database: return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, transaction=True, - time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def execute_sql(conn): + def _inner(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - def _inner(conn): - try: - if time_limit_ms is None: - return execute_sql(conn) - with sqlite_timelimit(conn, time_limit_ms): - return execute_sql(conn) - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - raise - with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn( _inner, block=block, request=request, transaction=transaction @@ -367,15 +354,6 @@ class Database: result = fn(self._write_connection) else: result = fn(self._write_connection) - if not block: - # There is no write thread here, so the write has already - # finished. Hand back the same (task_id, reply_future) shape - # _send_to_write_thread() returns, with the future already - # resolved, so the block=False path below is identical in - # both modes. - reply_future = asyncio.get_running_loop().create_future() - reply_future.set_result(result) - result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -447,7 +425,7 @@ class Database: ) self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid4() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() reply_future = loop.create_future() self._write_queue.put( @@ -781,17 +759,6 @@ class Database: return hidden_tables - async def derived_table_dependencies(self): - """Return implementation tables and the tables they derive from.""" - schema_version = (await self.execute("PRAGMA schema_version")).first()[0] - if ( - self._cached_derived_table_dependencies is None - or self._cached_derived_table_dependencies[0] != schema_version - ): - dependencies = await self.execute_fn(sqlite_derived_table_dependencies) - self._cached_derived_table_dependencies = (schema_version, dependencies) - return self._cached_derived_table_dependencies[1] - async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index 6def3698..f90a733e 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,17 +6,6 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType -_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) - - -def _normalize_http_url(value): - if not isinstance(value, str): - return None - normalized = value.strip() - if not _HTTP_URL_RE.fullmatch(normalized): - return None - return normalized - class UrlColumnType(ColumnType): name = "url" @@ -26,10 +15,7 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - normalized = _normalize_http_url(value) - if normalized is None: - return markupsafe.escape(value.strip()) - escaped = markupsafe.escape(normalized) + escaped = markupsafe.escape(value.strip()) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -37,7 +23,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if _normalize_http_url(value) is None: + if not re.match(r"^https?://\S+$", value.strip()): return "Invalid URL" return None diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index a4f5a4de..4494f07f 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -92,13 +92,6 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - # Resolve identity keys once per action, rather than scanning the - # restriction allowlist for every configured table's allow block. - self.restricted_table_keys = { - (db, self.action_obj.normalize_child(table) if self.action_obj else table) - for db, table in self.restricted_tables - } - def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: @@ -132,10 +125,8 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - child_key = ( - self.action_obj.normalize_child(child) if self.action_obj else child - ) - if (parent, child_key) in self.restricted_table_keys: + table_actions = table_restrictions.get(child, []) + if self.action_checks.intersection(table_actions): return True else: # Parent query should proceed if any child in this database is allowlisted diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index d30ebd3f..88e1d274 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -185,15 +185,11 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - action_obj = datasette.actions.get(action) - normalize = action_obj.normalize_child if action_obj else lambda name: name - for table_name, table_allowed in ( - restrictions.get("r", {}).get(database, {}).items() - ): - if normalize(table_name) == normalize(table): - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py deleted file mode 100644 index 11fd4008..00000000 --- a/datasette/default_permissions/sqlite_statistics.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Default table-access policy for SQLite optimizer statistics.""" - -import json - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - - -@hookimpl -def permission_resources_sql(action): - if action != "view-table": - return None - return PermissionSQL( - sql=""" - SELECT database_name AS parent, value AS child, 0 AS allow, - 'SQLite statistics tables are denied by default' AS reason - FROM catalog_databases - CROSS JOIN json_each(:sqlite_statistics_names) - """, - params={ - "sqlite_statistics_names": json.dumps( - ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] - ) - }, - ) diff --git a/datasette/filters.py b/datasette/filters.py index 0499d086..3cfb36e5 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,9 +1,8 @@ import json -import math from typing import ClassVar from datasette import hookimpl -from datasette.resources import DatabaseResource, TableResource +from datasette.resources import DatabaseResource from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError @@ -52,20 +51,13 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which trusted fts_table to use. Query string parameters can - # repeat this mapping (for backwards compatibility), but must not select - # a different table or primary key. + # Figure out which fts_table to use table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = table_metadata.get("fts_table") + fts_table = request.args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = table_metadata.get("fts_pk", "rowid") - requested_fts_table = request.args.get("_fts_table") - requested_fts_pk = request.args.get("_fts_pk") - if (requested_fts_table and requested_fts_table != fts_table) or ( - requested_fts_pk and requested_fts_pk != fts_pk - ): - raise BadRequest("Invalid _fts_table or _fts_pk") + fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) search_args = { key: request.args[key] for key in request.args @@ -83,11 +75,6 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=fts_table), - actor=request.actor, - ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -148,11 +135,6 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=through_table), - actor=request.actor, - ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) fk_to_us = next( @@ -203,17 +185,6 @@ class Filter: raise NotImplementedError -def _coerce_numeric_filter_value(value): - try: - return int(value) - except ValueError: - try: - converted = float(value) - except ValueError: - return value - return converted if math.isfinite(converted) else value - - class TemplatedFilter(Filter): def __init__( self, @@ -235,8 +206,8 @@ class TemplatedFilter(Filter): def where_clause(self, table, column, value, param_counter): converted = self.format.format(value) - if self.numeric: - converted = _coerce_numeric_filter_value(converted) + if self.numeric and converted.isdigit(): + converted = int(converted) if self.no_argument: kwargs = {"c": _quote_sqlite_identifier(column)} converted = None diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index ef6c7b7e..c36d5dbe 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -59,10 +59,6 @@ def handle_exception(datasette, request, exception): body = dict(info) body.update(error_body(plain_message or message, status)) return Response.json(body, status=status, headers=headers) - if request.path.split("?")[0].endswith(".csv"): - return Response.text( - plain_message or message, status=status, headers=headers - ) info.update( { "ok": False, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 0b807f8c..f89f2f36 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -9,11 +9,6 @@ def startup(datasette): """Fires directly after Datasette first starts running""" -@hookspec -def shutdown(datasette): - """Called once when the Datasette server is shutting down""" - - @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" diff --git a/datasette/permissions.py b/datasette/permissions.py index 2d242560..e03b065c 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -3,10 +3,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple -_SQLITE_IDENTIFIER_CASE = str.maketrans( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" -) - # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False @@ -53,15 +49,6 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables - case_insensitive_child: bool = False - - @classmethod - def normalize_child(cls, child: str | None) -> str | None: - """Return a comparison key without changing the resource's display name.""" - if cls.case_insensitive_child and child is not None: - # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. - return child.translate(_SQLITE_IDENTIFIER_CASE) - return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -159,11 +146,6 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed - def normalize_child(self, child: str | None) -> str | None: - if self.resource_class is None: - return child - return self.resource_class.normalize_child(child) - @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 6a4d7da7..9cf94079 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -18,7 +18,6 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", - "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", diff --git a/datasette/resources.py b/datasette/resources.py index 29bf7b1e..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,7 +25,6 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource - case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/app.css b/datasette/static/app.css index 234f535c..d101e4b7 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -216,49 +216,6 @@ a:active { text-decoration: underline; } -.table-summary .count-all ~ .table-summary-description { - margin-left: 0.5rem; -} - -.table-summary .count-error:not(:empty) { - display: block; - margin-top: 0.25rem; - font-size: 0.875rem; - font-weight: 400; - line-height: 1.5; -} - -button.count-all { - background: none; - border: none; - padding: 3px 0; - margin-left: 0.25rem; - color: #276890; - font-family: inherit; - font-size: 0.8125rem; - font-weight: 400; - line-height: 1.5; - cursor: pointer; -} - -button.count-all:hover, -button.count-all:focus-visible { - text-decoration: underline; -} - -button.count-all:disabled { - color: #596478; - cursor: wait; -} - -@media (pointer: coarse) { - button.count-all { - min-height: 44px; - padding-left: 7px; - padding-right: 7px; - } -} - button.button-as-link { background: none; border: none; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index c3d5796c..198641f3 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -472,13 +472,11 @@ class ColumnChooser extends HTMLElement { - + ${col}
`; - li.querySelector(".drag-item-label").textContent = col; - li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..0e6e2c29 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,56 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === "object" && typeof module !== "undefined" + ? (module.exports = factory()) + : typeof define === "function" && define.amd + ? define(factory) + : (global.jsonFormatHighlight = factory()); +})(this, function () { + "use strict"; + + var defaultColors = { + keyColor: "dimgray", + numberColor: "lightskyblue", + stringColor: "lightcoral", + trueColor: "lightseagreen", + falseColor: "#f66578", + nullColor: "cornflowerblue", + }; + + function index(json, colorOptions) { + if (colorOptions === void 0) colorOptions = {}; + + if (!json) { + return; + } + if (typeof json !== "string") { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, "&").replace(//g, ">"); + return json.replace( + /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, + function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) + ? colors.trueColor + : /false/.test(match) + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; + } + return '' + match + ""; + }, + ); + } + + return index; +}); diff --git a/datasette/static/table.js b/datasette/static/table.js index 143e976f..74a96d8e 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -860,45 +860,10 @@ function openColumnChooser() { }); } -function initCountAll() { - var button = document.querySelector(".count-all"); - if (!button) { - return; - } - button.addEventListener("click", async function () { - var count = document.querySelector(".table-count"); - var error = document.querySelector(".count-error"); - button.disabled = true; - button.textContent = "Counting…"; - error.textContent = ""; - try { - var response = await fetch(button.dataset.countUrl + location.search, { - method: "POST", - headers: { - Accept: "application/json", - }, - }); - var data = await response.json(); - if (!response.ok || !data.ok) { - throw new Error((data.errors || ["Count failed"]).join(" ")); - } - count.textContent = - data.count.toLocaleString("en-US") + - (data.count === 1 ? " row" : " rows"); - button.remove(); - } catch (ex) { - error.textContent = ex.message || "Count failed"; - button.disabled = false; - button.textContent = "count all"; - } - }); -} - // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; - initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 32686af1..4927cb8d 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,6 +3,7 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} + {% endblock %} {% block content %} @@ -125,7 +126,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -173,7 +174,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index c73cdfb7..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,6 +3,7 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -197,7 +198,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } function displayError(data) { @@ -207,7 +208,7 @@ function displayError(data) { resultsContent.innerHTML = `
Error: ${escapeHtml(data.error || 'Unknown error')}
`; - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index c0081c66..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,6 +3,7 @@ {% block title %}Explain a permission decision{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %}