diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index cf9b25a7..3fc83438 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,46 +14,24 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Check deployment prerequisites - id: deployment-prerequisites - env: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} - run: | - missing=() - for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do - if [[ -z "${!variable:-}" ]]; then - missing+=("$variable") - fi - done - if (( ${#missing[@]} )); then - echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" - echo "available=false" >> "$GITHUB_OUTPUT" - else - echo "available=true" >> "$GITHUB_OUTPUT" - fi - name: Check out datasette - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -61,18 +39,14 @@ jobs: fixtures-metadata.json \ plugins \ --extra-db-filename extra_database.db - # Package the config with the plugins, excluding test-only plugin secrets - # that reference temporary files outside the deployed container. - jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json - name: Build docs.db - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -84,7 +58,6 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Upload latest documentation database to S3 (only for main) - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - name: Deploy to docs as well (only for main) + if: ${{ github.ref == 'refs/heads/main' }} run: |- - # Keep development documentation separate from the stable release database. - s3-credentials put-object datasette-docs latest/docs.db docs.db \ - --content-type application/octet-stream + # Deploy docs.db to a different service + datasette publish cloudrun docs.db \ + --branch=$GITHUB_SHA \ + --version-note=$GITHUB_SHA \ + --extra-options="--setting template_debug 1" \ + --service=datasette-docs-latest diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 85369f6c..f5b8dbf6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,15 +2,9 @@ name: Playwright on: push: - branches: - - main pull_request: workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - permissions: contents: read diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index fa7ec6aa..d92ab82b 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -1,15 +1,6 @@ name: Check JavaScript for conformance with Prettier -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push] permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 232a34c7..21ed4c12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [published] + types: [created] permissions: contents: read @@ -51,8 +51,6 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 - # After the first non-prerelease 1.0 release, disable this job on 0.65.x, - # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -68,20 +66,26 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Upload stable documentation database to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Deploy stable-docs.datasette.io to Cloud Run run: |- - s3-credentials put-object datasette-docs docs.db docs.db \ - --content-type application/octet-stream + gcloud config set run/region us-central1 + gcloud config set project datasette-222320 + datasette publish cloudrun docs.db \ + --service=datasette-docs-stable deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index aa35338f..58635025 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -1,15 +1,6 @@ name: Check spelling in documentation -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 00000000..e9bd4bab --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,40 @@ +name: Calculate test coverage + +on: + push: + branches: + - main + pull_request: + branches: + - main +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out datasette + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . --group dev + python -m pip install pytest-cov + - name: Run tests + run: |- + ls -lah + cat .coveragerc + pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x + ls -lah + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 449855f3..5e81ed82 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -2,15 +2,9 @@ name: Test in Pyodide with shot-scraper on: push: - branches: - - main pull_request: workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - permissions: contents: read diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 700f3cce..2fdb3a40 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -1,15 +1,6 @@ name: Test SQLite versions -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read @@ -21,10 +12,10 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: ["3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] sqlite-version: [ #"3", # latest version - #"3.46", + "3.46", #"3.45", #"3.27", #"3.26", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8176a630..751eedfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,6 @@ name: Test -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +on: [push, pull_request] permissions: contents: read @@ -20,20 +11,16 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] - include: - - python-version: "3.14" - coverage: true + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) @@ -41,27 +28,12 @@ jobs: run: | pip install . --group dev pip freeze - - name: Install pytest-cov - if: ${{ matrix.coverage }} - run: pip install pytest-cov - name: Run tests run: | - if [ "${{ matrix.coverage }}" = "true" ]; then - COV="--cov=datasette --cov-config=.coveragerc" - pytest -n auto -m "not serial" $COV --cov-report= - pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term - else - pytest -n auto -m "not serial" - pytest -m "serial" - fi + pytest -n auto -m "not serial" + pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh - - name: Upload coverage report - if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - name: Black run: | black --version diff --git a/Dockerfile b/Dockerfile index 58287dd7..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm AS build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/README.md b/README.md index 1f79778f..393e8e5c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/datasette/__init__.py b/datasette/__init__.py index 982dcc79..e0022178 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,7 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor # noqa from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa from datasette.utils.asgi import ( # noqa Forbidden, diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 587380ed..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -89,8 +89,7 @@ def pytest_runtest_protocol(item, nextitem): continue try: ds.close() - except Exception as e: # noqa: BLE001 - # Surfaced as a pytest warning; teardown must not fail the run + except Exception as e: item.warn( pytest.PytestUnraisableExceptionWarning( f"Error closing Datasette instance: {e!r}" diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 7503f1d5..368213af 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,9 +1,7 @@ -import time - -from itsdangerous import BadSignature - from datasette import hookimpl +from itsdangerous import BadSignature from datasette.utils import baseconv +import time @hookimpl diff --git a/datasette/app.py b/datasette/app.py index a6998bef..4ba5d20f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,8 +2,7 @@ from __future__ import annotations import asyncio import contextvars -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence if TYPE_CHECKING: from datasette.permissions import Resource @@ -13,10 +12,11 @@ import dataclasses import datetime import functools import glob +import httpx import importlib.metadata import inspect +from itsdangerous import BadSignature import json -import logging import os import re import secrets @@ -28,42 +28,90 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx2 -from itsdangerous import BadSignature, URLSafeSerializer +from markupsafe import Markup, escape +from itsdangerous import URLSafeSerializer from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, - PrefixLoader, pass_context, + PrefixLoader, ) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -from markupsafe import Markup, escape -from . import stored_queries, write_sql -from .background_tasks import BackgroundTask, BackgroundTaskSupervisor -from .column_types import SQLiteType -from .csrf import CrossOriginProtectionMiddleware -from .database import Database, QueryInterrupted from .events import Event -from .plugins import DEFAULT_PLUGINS, get_plugins, pm +from .column_types import SQLiteType +from . import stored_queries, write_sql +from .views import Context +from .views.database import ( + database_download, + DatabaseView, + QueryView, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + QueryEditView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) +from .views.index import IndexView +from .views.special import ( + JsonDataView, + PatternPortfolioView, + AutocompleteDebugView, + AuthTokenView, + ApiExplorerView, + CreateTokenView, + LogoutView, + AllowDebugView, + PermissionsDebugView, + MessagesDebugView, + AllowedResourcesView, + PermissionRulesView, + PermissionCheckView, + JumpView, + InstanceSchemaView, + DatabaseSchemaView, + TableSchemaView, +) +from .views.table import ( + TableAutocompleteView, + TableInsertView, + TableUpsertView, + TableSetColumnTypeView, + TableDropView, + TableFragmentView, + table_view, +) +from .views.row import RowView, RowDeleteView, RowUpdateView from .renderer import json_renderer -from .resources import DatabaseResource, TableResource -from .tokens import TokenInvalid -from .tracer import AsgiTracer from .url_builder import Urls +from .database import Database, QueryInterrupted + from .utils import ( - SPATIALITE_FUNCTIONS, PaginatedResources, PrefixedUrlString, + SPATIALITE_FUNCTIONS, StartupError, - add_cors_headers, async_call_with_supported_arguments, await_me_maybe, baseconv, call_with_supported_arguments, detect_json1, + add_cors_headers, display_actor, escape_css_string, escape_sqlite, @@ -73,98 +121,47 @@ from .utils import ( move_plugins_and_allow, move_table_config, parse_metadata, - redact_keys, resolve_env_secrets, resolve_routes, - row_sql_params_pks, sha256_file, tilde_decode, tilde_encode, to_css_class, urlsafe_components, + redact_keys, + row_sql_params_pks, ) +from .tokens import TokenInvalid from .utils.asgi import ( AsgiLifespan, - AsgiRunOnFirstRequest, BadRequest, - DatabaseNotFound, Forbidden, NotFound, + DatabaseNotFound, + TableNotFound, + RowNotFound, Request, Response, - RowNotFound, - TableNotFound, + AsgiRunOnFirstRequest, + asgi_static, asgi_send, asgi_send_file, asgi_send_redirect, - asgi_static, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, ) +from .tracer import AsgiTracer +from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .views import Context -from .views.database import ( - DatabaseView, - QueryView, - database_download, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.index import IndexView -from .views.row import RowDeleteView, RowUpdateView, RowView -from .views.special import ( - AllowDebugView, - AllowedResourcesView, - ApiExplorerView, - AuthTokenView, - AutocompleteDebugView, - CreateTokenView, - DatabaseSchemaView, - InstanceSchemaView, - JsonDataView, - JumpView, - LogoutView, - MessagesDebugView, - PatternPortfolioView, - PermissionCheckView, - PermissionRulesView, - PermissionsDebugView, - TableSchemaView, -) -from .views.stored_queries import ( - GlobalQueryListView, - QueryCreateAnalyzeView, - QueryDefinitionView, - QueryDeleteView, - QueryEditView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) -from .views.table import ( - TableAutocompleteView, - TableCountView, - TableDropView, - TableFragmentView, - TableInsertView, - TableSetColumnTypeView, - TableUpsertView, - table_view, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) + +from .resources import DatabaseResource, TableResource app_root = Path(__file__).parent.parent -logger = logging.getLogger(__name__) - # Context variable to track when code is executing within a datasette.client request _in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) @@ -187,7 +184,7 @@ class PermissionCheck: """Represents a logged permission check for debugging purposes.""" when: str - actor: dict[str, Any] | None + actor: Dict[str, Any] | None action: str parent: str | None child: str | None @@ -317,7 +314,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action.name, parent, action.normalize_child(child)) + return (actor_key, action, parent, child) async def favicon(request, send): @@ -424,7 +421,6 @@ class Datasette: default_deny=False, ): self._startup_invoked = False - self._shutdown_invoked = False self._closed = False assert config_dir is None or isinstance( config_dir, Path @@ -438,7 +434,7 @@ class Datasette: if config_dir: db_files = [] for ext in ("db", "sqlite", "sqlite3"): - db_files.extend(config_dir.glob(f"*.{ext}")) + db_files.extend(config_dir.glob("*.{}".format(ext))) self.files += tuple(str(f) for f in db_files) if ( config_dir @@ -456,11 +452,8 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this - self._setup_db_done = False - self._suppress_background_tasks = False try: self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -468,10 +461,8 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() else: raise - self._background_tasks = BackgroundTaskSupervisor(self) self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: @@ -684,10 +675,10 @@ class Datasette: def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: - for hook_environment in pm.hook.jinja2_environment_from_request( + for environment in pm.hook.jinja2_environment_from_request( datasette=self, request=request, env=environment ): - environment = hook_environment + pass return environment def get_action(self, name_or_abbr: str): @@ -741,7 +732,7 @@ class Datasette: catalog_database_names.update( row["database_name"] for row in await internal_db.execute( - f"select distinct database_name from {table}" + "select distinct database_name from {}".format(table) ) if row["database_name"] is not None ) @@ -752,7 +743,7 @@ class Datasette: for stale_db_name in stale_databases: for table in catalog_table_names: conn.execute( - f"DELETE FROM {table} WHERE database_name = ?", + "DELETE FROM {} WHERE database_name = ?".format(table), [stale_db_name], ) @@ -762,7 +753,19 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - await populate_schema_tables(internal_db, db, schema_version) + placeholders = "(?, ?, ?, ?)" + values = [database_name, str(db.path), db.is_memory, schema_version] + if db.path is None: + placeholders = "(?, null, ?, ?)" + values = [database_name, db.is_memory, schema_version] + await internal_db.execute_write( + """ + INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) + VALUES {} + """.format(placeholders), + values, + ) + await populate_schema_tables(internal_db, db) @property def urls(self): @@ -801,13 +804,17 @@ class Datasette: action.name in action_names and action != action_names[action.name] ): - raise StartupError(f"Duplicate action name: {action.name}") + raise StartupError( + "Duplicate action name: {}".format(action.name) + ) if ( action.abbr and action.abbr in action_abbrs and action != action_abbrs[action.abbr] ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") + raise StartupError( + "Duplicate action abbr: {}".format(action.abbr) + ) action_names[action.name] = action if action.abbr: action_abbrs[action.abbr] = action @@ -866,7 +873,7 @@ class Datasette: actor_id: str, *, expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + restrictions: "TokenRestrictions | None" = None, handler: str | None = None, ) -> str: """ @@ -923,7 +930,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - name = next(iter(self.databases.keys())) + name = [key for key in self.databases.keys()][0] return self.databases[name] def add_database(self, db, name=None, route=None): @@ -936,7 +943,7 @@ class Datasette: suggestion = name i = 2 while name in self.databases: - name = f"{suggestion}_{i}" + name = "{}_{}".format(suggestion, i) i += 1 db.name = name db.route = route or name @@ -971,14 +978,13 @@ class Datasette: for db in dbs: try: db.close() - except Exception as e: # noqa: BLE001 - # Collect the first failure and re-raise after every close() has run + except Exception as e: if first_exception is None: first_exception = e if self.executor is not None: try: self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: # noqa: BLE001 + except Exception as e: if first_exception is None: first_exception = e if first_exception is not None: @@ -1327,15 +1333,24 @@ class Datasette: actual = ( actual_sqlite_type.value if actual_sqlite_type is not None - else f"unrecognized {column_detail.type!r}" + else "unrecognized {!r}".format(column_detail.type) ) raise ValueError( - f"Column type {ct_cls.name!r} is only applicable to SQLite types {allowed} but {database}.{resource}.{column} " - f"has SQLite type {actual}" + "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " + "has SQLite type {}".format( + ct_cls.name, + allowed, + database, + resource, + column, + actual, + ) ) async def _apply_column_types_config(self): """Load column_types from datasette.json config into the internal DB.""" + import logging + for db_name, db_conf in (self.config or {}).get("databases", {}).items(): for table_name, table_conf in db_conf.get("tables", {}).items(): for col_name, ct in table_conf.get("column_types", {}).items(): @@ -1345,7 +1360,7 @@ class Datasette: col_type = ct["type"] config = ct.get("config") if col_type not in self._column_types: - logger.warning( + logging.warning( "column_types config references unknown type %r " "for %s.%s.%s", col_type, @@ -1358,7 +1373,7 @@ class Datasette: db_name, table_name, col_name, col_type, config ) except ValueError as ex: - logger.warning(str(ex)) + logging.warning(str(ex)) async def get_column_type(self, database: str, resource: str, column: str): """ @@ -1411,7 +1426,7 @@ class Datasette: resource: str, column: str, column_type: str, - config: dict | None = None, + config: dict = None, ) -> None: """Assign a column type. Overwrites any existing assignment.""" ct_cls = self._column_types.get(column_type) @@ -1495,7 +1510,9 @@ class Datasette: possible_names = {plugin["name"], plugin["name"].replace("-", "_")} if plugin_name in possible_names: return _resolve_static_asset_path(plugin["static_path"], path) - raise FileNotFoundError(f"No static assets found for plugin {plugin_name}") + raise FileNotFoundError( + "No static assets found for plugin {}".format(plugin_name) + ) def _static_mounted_asset(self, mount_name, path): mount_name = mount_name.strip("/") @@ -1505,7 +1522,7 @@ class Datasette: _resolve_static_asset_path(dirname, path), self.urls.path("/{}/{}".format(mount_name, path.lstrip("/"))), ) - raise FileNotFoundError(f"No static mount found for {mount_name}") + raise FileNotFoundError("No static mount found for {}".format(mount_name)) def _static_asset_hash(self, filepath): filepath = Path(filepath) @@ -1537,28 +1554,15 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: - # Extension loading is only enabled for as long as it takes to - # load the configured extensions. Leaving it enabled would let - # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - try: - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - if sys.version_info >= (3, 12): - conn.load_extension(path, entrypoint=entrypoint) - else: - # Connection.load_extension() only gained the - # entrypoint argument in Python 3.12 - conn.execute( - "SELECT load_extension(?, ?)", [path, entrypoint] - ) - else: - conn.load_extension(extension) - finally: - conn.enable_load_extension(False) + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1609,17 +1613,18 @@ class Datasette: if await self.allowed(action="view-instance", actor=actor): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link - if database and await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), - actor=actor, - ): - crumbs.append( - { - "href": self.urls.database(database), - "label": database, - } - ) + if database: + if await self.allowed( + action="view-database", + resource=DatabaseResource(database=database), + actor=actor, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) # Table link if table: assert database, "table= requires database=" @@ -1638,7 +1643,7 @@ class Datasette: async def actors_from_ids( self, actor_ids: Iterable[str | int] - ) -> dict[int | str, dict]: + ) -> Dict[int | str, Dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1647,9 +1652,9 @@ class Datasette: return result async def track_event(self, event: Event): - assert isinstance( - event, self.event_classes - ), f"Invalid event type: {type(event)}" + assert isinstance(event, self.event_classes), "Invalid event type: {}".format( + type(event) + ) for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) @@ -1686,7 +1691,7 @@ class Datasette: self, actor: dict, action: str, - resource: Resource | None = None, + resource: "Resource" | None = None, ): """ Check if actor can see a resource and if it's private. @@ -1751,145 +1756,8 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) - if action == "view-table": - sql, params = await self._apply_derived_table_permissions_to_sql( - sql, - params, - actor=actor, - parent=parent, - include_is_private=include_is_private, - ) return ResourcesSQL(sql, params) - async def _allowed_derived_table_source( - self, database, source, *, actor, dependencies - ): - """Check an immediate source, denying sources that are themselves derived.""" - if any( - TableResource.normalize_child(table) - == TableResource.normalize_child(source) - for table in dependencies - ): - return False - # The source has no dependency in this map. Evaluate its own permission - # and prerequisites without starting another dependency check. - verdicts = await self._allowed_many( - actions=["view-table"], - resource=TableResource(database, source), - actor=actor, - check_derived=False, - ) - return verdicts["view-table"] - - async def _apply_derived_table_permissions_to_sql( - self, - sql, - params, - *, - actor, - parent, - include_is_private, - ): - databases = ( - [(parent, self.databases[parent])] - if parent in self.databases - else ([] if parent is not None else list(self.databases.items())) - ) - dependency_maps = dict( - zip( - (name for name, _ in databases), - await asyncio.gather( - *(db.derived_table_dependencies() for _, db in databases) - ), - ) - ) - dependencies = [ - (database_name, child, source) - for database_name, dependency_map in dependency_maps.items() - for child, source in dependency_map.items() - ] - if not dependencies: - return sql, params - - sources = sorted( - {(database_name, source) for database_name, _, source in dependencies} - ) - actor_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=actor, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - actor_allowed = dict(zip(sources, actor_verdicts)) - - anonymous_allowed = {} - if include_is_private: - anonymous_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=None, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - anonymous_allowed = dict(zip(sources, anonymous_verdicts)) - - wrapped_params = dict(params) - derived_rows = [ - [ - database_name, - child, - int(actor_allowed[(database_name, source)]), - *( - [int(anonymous_allowed[(database_name, source)])] - if include_is_private - else [] - ), - ] - for database_name, child, source in dependencies - ] - derived_param = "_datasette_derived_permissions" - while derived_param in wrapped_params: - derived_param += "_" - wrapped_params[derived_param] = json.dumps(derived_rows) - - derived_columns = "parent, child, source_allowed" - select_columns = "allowed.parent, allowed.child, allowed.reason" - if include_is_private: - derived_columns += ", source_anonymous_allowed" - select_columns += ( - ", CASE WHEN derived.source_anonymous_allowed = 0 " - "THEN 1 ELSE allowed.is_private END AS is_private" - ) - wrapped_sql = f""" -WITH derived_permissions({derived_columns}) AS ( - SELECT - json_extract(value, '$[0]'), - json_extract(value, '$[1]'), - json_extract(value, '$[2]') - {", json_extract(value, '$[3]')" if include_is_private else ""} - FROM json_each(:{derived_param}) -), -allowed AS ( -{sql} -) -SELECT {select_columns} -FROM allowed -LEFT JOIN derived_permissions AS derived - ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE -WHERE COALESCE(derived.source_allowed, 1) = 1 -ORDER BY allowed.parent, allowed.child -""".strip() - return wrapped_sql, wrapped_params - async def allowed_resources( self, action: str, @@ -2022,7 +1890,10 @@ ORDER BY allowed.parent, allowed.child if truncated and resources: last_resource = resources[-1] # Use tilde-encoding like table pagination - next_token = f"{tilde_encode(str(last_resource.parent))},{tilde_encode(str(last_resource.child))}" + next_token = "{},{}".format( + tilde_encode(str(last_resource.parent)), + tilde_encode(str(last_resource.child)), + ) return PaginatedResources( resources=resources, @@ -2040,7 +1911,7 @@ ORDER BY allowed.parent, allowed.child self, *, action: str, - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ) -> bool: """ @@ -2071,7 +1942,7 @@ ORDER BY allowed.parent, allowed.child self, *, actions: Sequence[str], - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ) -> dict[str, bool]: """ @@ -2092,17 +1963,11 @@ ORDER BY allowed.parent, allowed.child ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ - return await self._allowed_many( - actions=actions, resource=resource, actor=actor, check_derived=True - ) - - async def _allowed_many(self, *, actions, resource, actor, check_derived): - """Evaluate permissions, optionally applying the one-hop source policy.""" + from datasette.utils.actions_sql import check_permissions_for_actions from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, ) - from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None @@ -2135,7 +2000,7 @@ ORDER BY allowed.parent, allowed.child to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, self.actions[name], parent, child) + key = _permission_cache_key(actor, name, parent, child) if key in cache: final[name] = cache[key] continue @@ -2151,28 +2016,6 @@ ORDER BY allowed.parent, allowed.child child=child, ) - if ( - check_derived - and "view-table" in to_check - and raw.get("view-table") - and isinstance(resource, TableResource) - and parent in self.databases - ): - dependencies = await self.databases[parent].derived_table_dependencies() - source = next( - ( - source - for table, source in dependencies.items() - if TableResource.normalize_child(table) - == TableResource.normalize_child(child) - ), - None, - ) - if source is not None: - raw["view-table"] = await self._allowed_derived_table_source( - parent, source, actor=actor, dependencies=dependencies - ) - def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2190,9 +2033,7 @@ ORDER BY allowed.parent, allowed.child # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[ - _permission_cache_key(actor, self.actions[name], parent, child) - ] = final[name] + cache[_permission_cache_key(actor, name, parent, child)] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2215,7 +2056,7 @@ ORDER BY allowed.parent, allowed.child self, *, action: str, - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ): """ @@ -2269,15 +2110,13 @@ ORDER BY allowed.parent, allowed.child db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) # Find the foreign_key for this column - fk = next( - ( + try: + fk = [ foreign_key for foreign_key in foreign_keys if foreign_key["column"] == column - ), - None, - ) - if fk is None: + ][0] + except IndexError: return {} # Ensure user has permission to view the referenced table from datasette.resources import TableResource @@ -2365,17 +2204,16 @@ ORDER BY allowed.parent, allowed.child sqlite_extensions[extension] = result.fetchone()[0] else: sqlite_extensions[extension] = None - except Exception: # noqa: BLE001, S110 - # Probing for optional SQLite extensions - absence is the normal case + except Exception: pass # More details on SpatiaLite if "spatialite" in sqlite_extensions: spatialite_details = {} for fn in SPATIALITE_FUNCTIONS: try: - result = conn.execute(f"select {fn}()") + result = conn.execute("select {}()".format(fn)) spatialite_details[fn] = result.fetchone()[0] - except sqlite3.Error as e: + except Exception as e: spatialite_details[fn] = {"error": str(e)} sqlite_extensions["spatialite"] = spatialite_details @@ -2383,7 +2221,9 @@ ORDER BY allowed.parent, allowed.child fts_versions = [] for fts in ("FTS5", "FTS4", "FTS3"): try: - conn.execute(f"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)") + conn.execute( + "CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(fts=fts) + ) fts_versions.append(fts) except sqlite3.OperationalError: continue @@ -2442,7 +2282,7 @@ ORDER BY allowed.parent, allowed.child "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": sorted(set(p["hooks"])), + "hooks": list(sorted(set(p["hooks"]))), } for p in ps ] @@ -2466,21 +2306,6 @@ ORDER BY allowed.parent, allowed.child ) return d - def _tasks(self): - return { - "tasks": [ - { - "name": t.name, - "state": t.state, - "function": t.function, - "started_at": t.started_at, - "exception": repr(t.exception) if t.exception else None, - } - for t in self._background_tasks.tasks() - ], - "launched": self._background_tasks.launched, - } - def _actor(self, request): return {"actor": request.actor} @@ -2533,15 +2358,13 @@ ORDER BY allowed.parent, allowed.child async def render_template( self, - templates: list[str] | str | Template, - context: dict[str, Any] | Context | None = None, + templates: List[str] | str | Template, + context: Dict[str, Any] | Context | None = None, request: Request | None = None, view_name: str | None = None, ): if not self._startup_invoked: - raise RuntimeError( - "render_template() called before await ds.invoke_startup()" - ) + raise Exception("render_template() called before await ds.invoke_startup()") context = context or {} if isinstance(templates, Template): template = templates @@ -2587,11 +2410,9 @@ ORDER BY allowed.parent, allowed.child datasette=self, ): extra_vars = await await_me_maybe(extra_vars) - if extra_vars is None: - continue - assert isinstance( - extra_vars, dict - ), f"extra_vars is of type {type(extra_vars)}" + assert isinstance(extra_vars, dict), "extra_vars is of type {}".format( + type(extra_vars) + ) extra_template_vars.update(extra_vars) async def menu_links(): @@ -2610,27 +2431,29 @@ ORDER BY allowed.parent, allowed.child # the contract tests fail otherwise template_context = { **context, - "request": request, - "crumb_items": self._crumb_items, - "urls": self.urls, - "actor": request.actor if request else None, - "menu_links": menu_links, - "display_actor": display_actor, - "show_logout": request is not None - and "ds_actor" in request.cookies - and request.actor, - "zip": zip, - "body_scripts": body_scripts, - "format_bytes": format_bytes, - "show_messages": lambda: self._show_messages(request), - "extra_css_urls": await self._asset_urls( - "extra_css_urls", template, context, request, view_name - ), - "extra_js_urls": await self._asset_urls( - "extra_js_urls", template, context, request, view_name - ), - "base_url": self.setting("base_url"), - "datasette_version": __version__, + **{ + "request": request, + "crumb_items": self._crumb_items, + "urls": self.urls, + "actor": request.actor if request else None, + "menu_links": menu_links, + "display_actor": display_actor, + "show_logout": request is not None + and "ds_actor" in request.cookies + and request.actor, + "zip": zip, + "body_scripts": body_scripts, + "format_bytes": format_bytes, + "show_messages": lambda: self._show_messages(request), + "extra_css_urls": await self._asset_urls( + "extra_css_urls", template, context, request, view_name + ), + "extra_js_urls": await self._asset_urls( + "extra_js_urls", template, context, request, view_name + ), + "base_url": self.setting("base_url"), + "datasette_version": __version__, + }, **extra_template_vars, } if request and request.args.get("_context") and self.setting("template_debug"): @@ -2651,7 +2474,7 @@ ORDER BY allowed.parent, allowed.child ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + expire_after + expires_at = int(time.time()) + (24 * 60 * 60) data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2752,7 +2575,7 @@ ORDER BY allowed.parent, allowed.child JsonDataView.as_view( self, "plugins.json", - self._plugins, + lambda request: {"plugins": self._plugins(request)}, needs_request=True, ), r"/-/plugins(\.(?Pjson))?$", @@ -2771,12 +2594,6 @@ ORDER BY allowed.parent, allowed.child ), r"/-/threads(\.(?Pjson))?$", ) - add_route( - JsonDataView.as_view( - self, "tasks.json", self._tasks, permission="permissions-debug" - ), - r"/-/tasks(\.(?Pjson))?$", - ) add_route( JsonDataView.as_view( self, @@ -2951,10 +2768,6 @@ ORDER BY allowed.parent, allowed.child TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) - add_route( - TableCountView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", - ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -3018,127 +2831,26 @@ ORDER BY allowed.parent, allowed.child raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) - async def _startup_sequence(self): - """Idempotently run the full startup sequence: table counts for - immutable databases, then invoke_startup(). Safe to call more than - once and safe to call concurrently - callers block until whichever - call got there first has finished. - - This is the single entry point used by both AsgiLifespan (so - real deployments finish startup before accepting requests) and - AsgiRunOnFirstRequest (the fallback for hosts that never send - lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and - `datasette serve` (cli.py) calls it too. The fast path below checks - both `_startup_invoked` and `_setup_db_done` - not just the former - - so that a bare `await ds.invoke_startup()` made by a caller ahead of - `_startup_sequence()` (which only sets `_startup_invoked`) can't - make this method skip the immutable-database table-count precompute. - """ - if self._startup_invoked and self._setup_db_done: - return - async with self._startup_lock: - if self._startup_invoked and self._setup_db_done: - return - if not self._setup_db_done: - # First time server starts up, calculate table counts for - # immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - self._setup_db_done = True - await self.invoke_startup() - - def add_background_task(self, func, name=None) -> BackgroundTask: - """Register a piece of supervised background work, typically from - a plugin's ``startup`` hook. - - ``func`` must be a coroutine function taking one positional - argument, the ``Datasette`` instance - core calls ``func(self)``. - Callable any time after ``__init__``: if background tasks haven't - launched yet (the common case - most callers are ``startup`` hooks, - which run before launch), this buffers the registration until they - do; if they've already launched (e.g. called from a request - handler after the server is up), the task starts immediately. - - Returns a :class:`~datasette.background_tasks.BackgroundTask` - handle (``.name``, ``.state``, ``.task``, ``.exception``, - ``.started_at``, ``.function``, ``.cancel()``). - - ``name`` defaults to ``func.__qualname__``; on a name collision a - ``-2``, ``-3``, ... suffix is appended, since names are how - ``/-/tasks`` and log messages identify work. - """ - return self._background_tasks.add(func, name=name) - - async def start_background_tasks(self): - """Run startup (if it hasn't run yet) and launch every registered - background task. - - Public entry point for tests, embedders, and headless CLIs (the - ``datasette-rss``-style ``fetch --due`` shape) that want supervised - background tasks without running a server - equivalent to what - happens automatically via ASGI lifespan / the first-request - fallback in a served deployment. - """ - await self.invoke_startup() - await self._background_tasks.launch_all() - - async def _launch_background_tasks(self): - """Idempotently launch every registered background task. Private: - this is the entry point wired into the lifecycle trigger lists - (the second entry in both ``AsgiLifespan`` and - ``AsgiRunOnFirstRequest``'s ``on_startup``, after - ``_startup_sequence``) - not something plugins or embedders should - call directly; use ``add_background_task`` / - ``start_background_tasks`` instead. - - Positioned after ``_startup_sequence`` in both trigger lists so - launch always happens once every plugin's ``startup`` hook has had - a chance to register work - the ordering guarantee that makes - ``add_background_task`` useful. No-ops when - ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its - one-shot TestClient request flows through the full ASGI stack, - including the first-request fallback, but must never launch - long-lived background work). - """ - if self._suppress_background_tasks: - return - await self._background_tasks.launch_all() - - async def invoke_shutdown(self): - """Run the graceful teardown sequence: plugin ``shutdown`` hooks, - then cancel and drain supervised background tasks, then close - every database. - """ - if self._shutdown_invoked: - return - self._shutdown_invoked = True - for hook in pm.hook.shutdown(datasette=self): - try: - await await_me_maybe(hook) - except Exception: - logging.getLogger("datasette").exception("shutdown hook failed") - await self._background_tasks.cancel_all(grace=5.0) - self.close() - def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() + async def setup_db(): + # First time server starts up, calculate table counts for immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + + async def _close_on_shutdown(): + self.close() + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan( - asgi, - on_startup=[self._startup_sequence, self._launch_background_tasks], - on_shutdown=[self.invoke_shutdown], - ) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) - asgi = AsgiRunOnFirstRequest( - asgi, - on_startup=[self._startup_sequence, self._launch_background_tasks], - ) return asgi @@ -3176,50 +2888,6 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) - match, view = resolve_routes(self.routes, path) - is_static = view is favicon or getattr(view, "_datasette_static", False) - original_send = send - - async def send(message): - if message["type"] == "http.response.start" and not ( - is_static and message["status"] in (200, 304) - ): - # Decide privacy after rendering, including for streaming responses - # and error handlers. A public primary resource can still include - # private labels, actor navigation, or cookie-dependent content. - headers = list(message.get("headers", [])) - personalized = ( - request.actor is not None - or "cookie" in request.headers - or "authorization" in request.headers - or any(key.lower() == b"set-cookie" for key, _ in headers) - ) - if personalized: - headers = [ - (key, value) - for key, value in headers - if key.lower() != b"cache-control" - ] - headers.append((b"cache-control", b"private, no-store")) - - # Anonymous responses must not be reused for credentialed requests. - # Preserve any additional variation specified by views or plugins. - vary = [ - part.strip() - for key, value in headers - if key.lower() == b"vary" - for part in value.split(b",") - if part.strip() - ] - if b"*" not in vary: - for name in (b"Cookie", b"Authorization"): - if name.lower() not in {part.lower() for part in vary}: - vary.append(name) - headers = [(k, v) for k, v in headers if k.lower() != b"vary"] - headers.append((b"vary", b", ".join(vary))) - message = dict(message, headers=headers) - await original_send(message) - # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -3259,7 +2927,8 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - request.scope = scope + + match, view = resolve_routes(self.routes, path) if match is None: return await self.handle_404(request, send) @@ -3284,8 +2953,7 @@ class DatasetteRouter: custom_response ), "Default forbidden() hook should have been called" return await custom_response.asgi_send(send) - except Exception as exception: # noqa: BLE001 - # This IS the top-level error handler - it must catch everything + except Exception as exception: return await self.handle_exception(request, send, exception) async def handle_401(self, request, send, exception): @@ -3307,7 +2975,7 @@ class DatasetteRouter: request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E") ) if request.query_string: - new_path += f"?{request.query_string}" + new_path += "?{}".format(request.query_string) await asgi_send_redirect(send, new_path) return # If URL has a trailing slash, redirect to URL without it @@ -3517,7 +3185,8 @@ _curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): # Drop the ".html" suffix - filepath = filepath.removesuffix(".html") + if filepath.endswith(".html"): + filepath = filepath[: -len(".html")] re_bits = ["/"] for bit in _curly_re.split(filepath): if _curly_re.match(bit): @@ -3574,14 +3243,14 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) else: - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await getattr(client, method)(self._fix(path), **kwargs) @@ -3628,10 +3297,10 @@ class DatasetteClient: method: HTTP method (e.g., "GET", "POST", "PUT") path: The path to request skip_permission_checks: If True, bypass all permission checks for this request - **kwargs: Additional arguments to pass to httpx2 + **kwargs: Additional arguments to pass to httpx Returns: - httpx2.Response: The response from the request + httpx.Response: The response from the request """ from datasette.permissions import SkipPermissions @@ -3640,16 +3309,16 @@ class DatasetteClient: with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) else: - async with httpx2.AsyncClient( - transport=httpx2.ASGITransport(app=self.app), + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: return await client.request( diff --git a/datasette/background_tasks.py b/datasette/background_tasks.py deleted file mode 100644 index 6b34fd85..00000000 --- a/datasette/background_tasks.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Supervised background-task registration for Datasette core. - -Plugins that need long-lived background work (a polling loop, a queue -consumer, a scheduled job runner) register it with -``datasette.add_background_task(func, name=None)`` - typically from a -``startup`` plugin hook - instead of fire-and-forgetting their own -``asyncio.create_task()``. Core owns: - -- **references**: every launched ``asyncio.Task`` is kept alive on a - :class:`BackgroundTaskSupervisor`, so it can never be silently garbage - collected the way an unreferenced ``create_task()`` call can be; -- **launch timing**: registered work is buffered until - :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to - happen only after *every* plugin's ``startup`` hook has finished - so - a task that depends on another plugin having registered something first - doesn't need ``tryfirst=True`` ordering tricks; -- **crash surfacing**: an unhandled exception in a background task is - logged with its full traceback to the ``datasette.background_tasks`` - logger and recorded on the handle, instead of becoming an "Task - exception was never retrieved" warning nobody sees; -- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels - every task still running and waits (with a grace period) for them to - actually stop. -""" - -from __future__ import annotations - -import asyncio -import datetime -import functools -import logging -from collections.abc import Awaitable, Callable - -logger = logging.getLogger("datasette.background_tasks") - - -def _utcnow_iso() -> str: - return datetime.datetime.now(datetime.timezone.utc).isoformat() - - -def _function_path(func: Callable) -> str: - """Describe the callable without guessing which plugin registered it.""" - while isinstance(func, functools.partial): - func = func.func - if not hasattr(func, "__qualname__"): - func = type(func).__call__ - return f"{func.__module__}.{func.__qualname__}" - - -class BackgroundTask: - """A handle to a single piece of supervised background work. - - States: ``registered`` (added but not yet launched) -> ``running`` -> - one of ``completed`` (returned cleanly), ``crashed`` (raised an - exception other than ``CancelledError`` - see ``.exception``), or - ``cancelled`` (``.cancel()`` was called, or it was still running at - shutdown). - """ - - def __init__( - self, - name: str, - func: Callable[[object], Awaitable[None]], - ): - self.name = name - self.state = "registered" - self.task: asyncio.Task | None = None - self.exception: BaseException | None = None - self.started_at: str | None = None - self.function = _function_path(func) - self._func = func - self._supervisor: BackgroundTaskSupervisor | None = None - - def cancel(self) -> None: - """Cancel this task. - - If it has already been launched, cancels the underlying - ``asyncio.Task`` - its state becomes ``cancelled`` once the - cancellation is observed (asynchronously, via the task's done - callback). If it has not been launched yet, this is a no-op as - far as asyncio is concerned (there's no task to cancel) but it - deregisters the handle from its supervisor so it never runs. - """ - if self.task is not None: - self.task.cancel() - elif self._supervisor is not None: - self._supervisor._deregister(self) - - def __repr__(self) -> str: - return f"" - - -class BackgroundTaskSupervisor: - """Owns registration and launch of every :class:`BackgroundTask` for a - single ``Datasette`` instance. - - Registration (:meth:`add`) is separate from launch - (:meth:`launch_all`): plugins register work whenever convenient - (typically from a ``startup`` hook, but request handlers can register - dynamic per-job work too), and it either sits buffered until - :meth:`launch_all` runs, or - if :meth:`launch_all` has already run - - starts immediately. - - Strong references to every :class:`BackgroundTask` (and its - ``asyncio.Task``) are kept for the life of the instance, by design - - that's what makes the enrichments-style "fire-and-forget task gets - garbage collected mid-flight" bug impossible here. There is currently - no pruning of completed/crashed/cancelled tasks, so a plugin that - dynamically registers many short-lived tasks over a long process - lifetime (a per-job registration pattern, e.g. one task per queued - job) will grow this list without bound. That's an accepted v1 - trade-off in favour of full introspection (``/-/tasks``); revisit - with a pruning or capping policy if unbounded growth is reported in - practice. - """ - - def __init__(self, datasette): - self._datasette = datasette - self._tasks: list[BackgroundTask] = [] - self._names = set() - self._launched = False - self._lock = asyncio.Lock() - - def add(self, func, name=None) -> BackgroundTask: - base_name = name or getattr(func, "__qualname__", None) or repr(func) - actual_name = self._unique_name(base_name) - handle = BackgroundTask(actual_name, func) - handle._supervisor = self - self._tasks.append(handle) - self._names.add(actual_name) - if self._launched: - self._launch_one(handle) - return handle - - def _unique_name(self, base_name: str) -> str: - if base_name not in self._names: - return base_name - n = 2 - while f"{base_name}-{n}" in self._names: - n += 1 - return f"{base_name}-{n}" - - def _deregister(self, handle: BackgroundTask) -> None: - try: - self._tasks.remove(handle) - except ValueError: - pass - self._names.discard(handle.name) - - def _launch_one(self, handle: BackgroundTask) -> None: - handle.state = "running" - handle.started_at = _utcnow_iso() - handle.task = asyncio.create_task( - handle._func(self._datasette), name=handle.name - ) - handle.task.add_done_callback(functools.partial(_on_task_done, handle)) - - async def launch_all(self) -> None: - """Launch every currently-registered task that hasn't launched - yet. Idempotent and safe to call concurrently: subsequent (or - racing) calls are no-ops once the first has set ``self._launched``. - """ - if self._launched: - return - async with self._lock: - if self._launched: - return - self._launched = True - for handle in list(self._tasks): - if handle.task is None: - self._launch_one(handle) - - async def cancel_all(self, grace: float = 5.0) -> None: - """Cancel every task that isn't already done, then wait up to - ``grace`` seconds for them to actually finish. Stragglers still - running after that are logged by name (but left to finish or not - on their own - this does not forcibly kill them, asyncio has no - mechanism for that). - """ - handles_by_task = { - handle.task: handle for handle in self._tasks if handle.task is not None - } - pending = [task for task in handles_by_task if not task.done()] - for task in pending: - task.cancel() - if not pending: - return - _done, not_done = await asyncio.wait(pending, timeout=grace) - if not_done: - names = sorted(handles_by_task[task].name for task in not_done) - logger.warning( - "%d background task(s) did not finish within the %.1fs grace " - "period after cancellation: %s", - len(names), - grace, - ", ".join(names), - ) - - def tasks(self) -> list[BackgroundTask]: - """Return every registered :class:`BackgroundTask`, launched or - not, in registration order. Used by the ``/-/tasks`` debug - endpoint. - """ - return list(self._tasks) - - @property - def launched(self) -> bool: - """Whether :meth:`launch_all` has run yet - lets ``/-/tasks`` - distinguish "no tasks registered" from "tasks registered but - nothing has armed the launch yet" without reaching for the - private ``_launched`` attribute. - """ - return self._launched - - -def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None: - if task.cancelled(): - handle.state = "cancelled" - return - exc = task.exception() - if exc is not None: - handle.state = "crashed" - handle.exception = exc - logger.error("Background task %r crashed", handle.name, exc_info=exc) - return - handle.state = "completed" diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index b6c8b77f..4d8c6bea 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -1,8 +1,7 @@ -import hashlib - from datasette import hookimpl +from datasette.utils.asgi import Response, BadRequest from datasette.utils import to_css_class -from datasette.utils.asgi import BadRequest, Response +import hashlib _BLOB_COLUMN = "_blob_column" _BLOB_HASH = "_blob_hash" diff --git a/datasette/cli.py b/datasette/cli.py index 7887e642..90a33e80 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -1,45 +1,43 @@ import asyncio +import uvicorn +import click +from click import formatting +from click.types import CompositeParamType +from click_default_group import DefaultGroup import functools import json import os import pathlib +from runpy import run_module import shutil +from subprocess import call import sys import textwrap import webbrowser -from runpy import run_module -from subprocess import call - -import click -import uvicorn -from click import formatting -from click.types import CompositeParamType -from click_default_group import DefaultGroup - from .app import ( + Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, - Datasette, pm, ) from .inspect import inspect_tables from .utils import ( - ConnectionProblem, LoadExtension, - SpatialiteConnectionProblem, - SpatialiteNotFound, StartupError, - StaticMount, - ValueAsBooleanError, check_connection, deep_dict_update, find_spatialite, + parse_metadata, + ConnectionProblem, + SpatialiteConnectionProblem, initial_path_for_datasette, pairs_to_nested_config, - parse_metadata, temporary_docker_directory, value_as_boolean, + SpatialiteNotFound, + StaticMount, + ValueAsBooleanError, ) from .utils.sqlite import sqlite3 from .utils.testing import TestClient @@ -77,7 +75,7 @@ class Setting(CompositeParamType): # Datasette 1.0, we turn bare setting names into setting.name # Type checking for those older settings default = DEFAULT_SETTINGS[name] - name = f"settings.{name}" + name = "settings.{}".format(name) if isinstance(default, bool): try: return name, "true" if value_as_boolean(value) else "false" @@ -173,6 +171,7 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" + pass # Register publish plugins @@ -497,7 +496,6 @@ def uninstall(packages, yes): "--internal", type=click.Path(), help="Path to a persistent Datasette internal SQLite database", - envvar="DATASETTE_INTERNAL", ) def serve( files, @@ -580,27 +578,27 @@ def serve( # https://github.com/simonw/datasette/issues/2389 deep_dict_update(config_data, settings_updates) - kwargs = { - "immutables": immutable, - "cache_headers": not reload, - "cors": cors, - "inspect_data": inspect_data, - "config": config_data, - "metadata": metadata_data, - "sqlite_extensions": sqlite_extensions, - "template_dir": template_dir, - "plugins_dir": plugins_dir, - "static_mounts": static, - "settings": None, # These are passed in config= now - "memory": memory, - "secret": secret, - "version_note": version_note, - "pdb": pdb, - "crossdb": crossdb, - "nolock": nolock, - "internal": internal, - "default_deny": default_deny, - } + kwargs = dict( + immutables=immutable, + cache_headers=not reload, + cors=cors, + inspect_data=inspect_data, + config=config_data, + metadata=metadata_data, + sqlite_extensions=sqlite_extensions, + template_dir=template_dir, + plugins_dir=plugins_dir, + static_mounts=static, + settings=None, # These are passed in config= now + memory=memory, + secret=secret, + version_note=version_note, + pdb=pdb, + crossdb=crossdb, + nolock=nolock, + internal=internal, + default_deny=default_deny, + ) # Separate directories from files directories = [f for f in files if os.path.isdir(f)] @@ -623,7 +621,9 @@ def serve( conn.close() else: raise click.ClickException( - f"Invalid value for '[FILES]...': Path '{file}' does not exist." + "Invalid value for '[FILES]...': Path '{}' does not exist.".format( + file + ) ) # Check for duplicate files by resolving all paths to their absolute forms @@ -664,6 +664,16 @@ def serve( # Private utility mechanism for writing unit tests return ds + # Run async soundness checks before startup hooks, since invoke_startup + # now populates internal tables which requires querying each database + run_sync(lambda: check_databases(ds)) + + # Run the "startup" plugin hooks + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -671,23 +681,10 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: - # --get means we don't run Uvicorn at all - run_sync(lambda: check_databases(ds)) - - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - - # --get never launches background tasks: TestClient's request below - # flows through the full ASGI stack, including the - # AsgiRunOnFirstRequest fallback, which would otherwise launch them. - ds._suppress_background_tasks = True - client = TestClient(ds) request_headers = {} if token: - request_headers["Authorization"] = f"Bearer {token}" + request_headers["Authorization"] = "Bearer {}".format(token) cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) @@ -708,54 +705,30 @@ def serve( sys.exit(exit_code) return - # check_databases, invoke_startup() and the uvicorn server all run on a - # single event loop, so that anything a plugin's "startup" hook schedules - # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is - # still alive when the server starts handling requests. - async def _serve_async(): - # Populate internal catalog tables before invoke_startup - await check_databases(ds) - - # Run the full startup sequence (immutable-database table-count - # precompute + the "startup" plugin hooks) via the same entry point - # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when - # uvicorn's lifespan.startup fires moments later. - try: - await ds._startup_sequence() - except StartupError as e: - raise click.ClickException(e.args[0]) - - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = await initial_path_for_datasette(ds) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) - await server.serve() - - asyncio.run(_serve_async()) + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = run_sync(lambda: initial_path_for_datasette(ds)) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = dict( + host=host, port=port, log_level="info", lifespan="on", workers=1 + ) + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + uvicorn.run(ds.app(), **uvicorn_kwargs) @cli.command() @@ -912,7 +885,7 @@ async def check_databases(ds): ) except ConnectionProblem as e: raise click.UsageError( - f"Connection to {database.path} failed check: {e.args[0]!s}" + f"Connection to {database.path} failed check: {str(e.args[0])}" ) # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning if ( @@ -920,5 +893,9 @@ async def check_databases(ds): and len([db for db in ds.databases.values() if not db.is_memory]) > SQLITE_LIMIT_ATTACHED ): - msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases" + msg = ( + "Warning: --crossdb only works with the first {} attached databases".format( + SQLITE_LIMIT_ATTACHED + ) + ) click.echo(click.style(msg, bold=True, fg="yellow"), err=True) diff --git a/datasette/column_types.py b/datasette/column_types.py index 92fdd969..11a14ec0 100644 --- a/datasette/column_types.py +++ b/datasette/column_types.py @@ -64,14 +64,14 @@ class ColumnType: Return an HTML string to render this cell value, or None to fall through to the default render_cell plugin hook chain. """ - return + return None async def validate(self, value, datasette): """ Validate a value before it is written. Return None if valid, or a string error message if invalid. """ - return + return None async def transform_value(self, value, datasette): """ diff --git a/datasette/csrf.py b/datasette/csrf.py index a62f9473..df239aee 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -40,12 +40,12 @@ def _origin_tuple(value): scheme = (parsed.scheme or "").lower() host = (parsed.hostname or "").lower() if not scheme or not host: - raise ValueError(f"missing scheme or host in {value!r}") + raise ValueError("missing scheme or host in {!r}".format(value)) port = parsed.port # may raise ValueError on bad ports if port is None: port = DEFAULT_PORTS.get(scheme) if port is None: - raise ValueError(f"unknown default port for scheme {scheme!r}") + raise ValueError("unknown default port for scheme {!r}".format(scheme)) return scheme, host, port @@ -125,7 +125,9 @@ class CrossOriginProtectionMiddleware: return await self._forbid( send, - f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'", + "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( + sec_fetch_site + ), ) return @@ -139,11 +141,11 @@ class CrossOriginProtectionMiddleware: request_scheme = self._request_scheme(scope) try: origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple(f"{request_scheme}://{host}") + expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) except ValueError: await self._forbid( send, - f"Malformed Origin {origin!r} or Host {host!r}", + "Malformed Origin {!r} or Host {!r}".format(origin, host), ) return @@ -153,7 +155,7 @@ class CrossOriginProtectionMiddleware: await self._forbid( send, - f"Origin {origin!r} does not match Host {host!r}", + "Origin {!r} does not match Host {!r}".format(origin, host), ) def _request_scheme(self, scope): @@ -161,8 +163,7 @@ class CrossOriginProtectionMiddleware: try: if self.datasette.setting("force_https_urls"): return "https" - except Exception: # noqa: BLE001, S110 - # Settings may not be readable this early; fall back to the ASGI scheme + except Exception: pass return scope.get("scheme") or "http" diff --git a/datasette/database.py b/datasette/database.py index 024f62a4..e7fe1ed9 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,35 +1,33 @@ import asyncio import atexit +from collections import namedtuple import inspect import os +from pathlib import Path import queue +import sqlite_utils import sys import tempfile import threading import uuid -from collections import namedtuple -from pathlib import Path -import sqlite_utils - -from .inspect import inspect_hash from .tracer import trace from .utils import ( call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, - escape_sqlite, get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, - sqlite3, sqlite_timelimit, - table_column_details, + sqlite3, table_columns, + table_column_details, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names +from .utils.sqlite import sqlite_hidden_table_names +from .inspect import inspect_hash connections = threading.local() @@ -85,7 +83,6 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None - self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -94,15 +91,16 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - # Track file and memory connections, including reads on worker threads, - # so close() can release all of them from the calling thread. - self._all_connections = [] + # This is used to track all file connections so they can be closed + self._all_file_connections = [] if not is_temp_disk: self.mode = mode def _check_not_closed(self): if self._closed: - raise DatasetteClosedError(f"Database {self.name!r} has been closed") + raise DatasetteClosedError( + "Database {!r} has been closed".format(self.name) + ) def _remove_pending_execute_future(self, future): with self._pending_execute_futures_lock: @@ -141,18 +139,15 @@ class Database: if write: extra_kwargs["isolation_level"] = "IMMEDIATE" if self.memory_name: - uri = f"file:{self.memory_name}?mode=memory&cache=shared" + uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) conn = sqlite3.connect( uri, uri=True, check_same_thread=False, **extra_kwargs ) if not write: conn.execute("PRAGMA query_only=1") - self._all_connections.append(conn) return conn if self.is_memory: - conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False) - self._all_connections.append(conn) - return conn + return sqlite3.connect(":memory:", uri=True) # mode=ro or immutable=1? if self.is_mutable: @@ -169,7 +164,7 @@ class Database: conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) - self._all_connections.append(conn) + self._all_file_connections.append(conn) if self.is_temp_disk and not self._wal_enabled: conn.execute("PRAGMA journal_mode=WAL") self._wal_enabled = True @@ -197,22 +192,23 @@ class Database: write_thread.join(timeout=10) if write_thread.is_alive(): sys.stderr.write( - f"Datasette: write thread for {self.name!r} did not exit within 10s\n" + "Datasette: write thread for {!r} did not exit within 10s\n".format( + self.name + ) ) sys.stderr.flush() for future in pending_execute_futures: try: future.result() - except Exception: # noqa: BLE001, S110 - # Shutdown teardown - a failed pending write must not block close() + except Exception: pass - # Close anything still tracked in _all_connections - for connection in self._all_connections: + # Close anything still tracked in _all_file_connections + for connection in self._all_file_connections: try: connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass - self._all_connections = [] + self._all_file_connections = [] # Drop per-thread cached read connections we can reach try: delattr(connections, self._thread_local_id) @@ -222,13 +218,13 @@ class Database: if self._read_connection is not None: try: self._read_connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._read_connection = None if self._write_connection is not None: try: self._write_connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._write_connection = None if self.is_temp_disk: @@ -250,34 +246,19 @@ class Database: request=None, return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, - transaction=True, - time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def execute_sql(conn): + def _inner(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - def _inner(conn): - try: - if time_limit_ms is None: - return execute_sql(conn) - with sqlite_timelimit(conn, time_limit_ms): - return execute_sql(conn) - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - raise - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn( - _inner, block=block, request=request, transaction=transaction - ) + results = await self.execute_write_fn(_inner, block=block, request=request) return results async def execute_write_script(self, sql, block=True, request=None): @@ -328,9 +309,9 @@ class Database: finally: isolated_connection.close() try: - self._all_connections.remove(isolated_connection) + self._all_file_connections.remove(isolated_connection) except ValueError: - # May already have been cleared by close(). + # Was probably a memory connection pass if self.ds.executor is None: @@ -367,19 +348,9 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: - self._write_connection.execute("BEGIN IMMEDIATE") result = fn(self._write_connection) else: result = fn(self._write_connection) - if not block: - # There is no write thread here, so the write has already - # finished. Hand back the same (task_id, reply_future) shape - # _send_to_write_thread() returns, with the future already - # resolved, so the block=False path below is identical in - # both modes. - reply_future = asyncio.get_running_loop().create_future() - reply_future.set_result(result) - result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -395,8 +366,7 @@ class Database: async def _dispatch_events_after_write(): try: await reply_future - except Exception: # noqa: BLE001 - # The write failed; skip success events regardless of why + except Exception: # if the write failed, don't emit success events return for event in pending_events: @@ -449,9 +419,11 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) - self._write_thread.name = f"_execute_writes for database {self.name}" + self._write_thread.name = "_execute_writes for database {}".format( + self.name + ) self._write_thread.start() - task_id = uuid.uuid4() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() reply_future = loop.create_future() self._write_queue.put( @@ -470,8 +442,7 @@ class Database: try: conn = self.connect(write=True) self.ds._prepare_connection(conn, self.name) - except Exception as e: # noqa: BLE001 - # Stored and re-raised to whoever queues the next write + except Exception as e: conn_exception = e while True: task = self._write_queue.get() @@ -479,8 +450,7 @@ class Database: if conn is not None: try: conn.close() - except Exception: # noqa: BLE001, S110 - # Best-effort close as the write thread exits + except Exception: pass return exception = None @@ -495,25 +465,23 @@ class Database: finally: isolated_connection.close() try: - self._all_connections.remove(isolated_connection) + self._all_file_connections.remove(isolated_connection) except ValueError: - # May already have been cleared by close(). + # Was probably a memory connection pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e else: try: if task.transaction: with conn: - conn.execute("BEGIN IMMEDIATE") result = task.fn(conn) else: result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e _deliver_write_result(task, result, exception) @@ -580,7 +548,9 @@ class Database: raise QueryInterrupted(e, sql, params) if log_sql_errors: sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + "ERROR: conn={}, sql = {}, params = {}: {}\n".format( + conn, repr(sql), params, e + ) ) sys.stderr.flush() raise @@ -633,7 +603,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})", + f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", custom_time_limit=limit, ) ).rows[0][0] @@ -737,9 +707,9 @@ class Database: column_names and len(column_names) == 2 and ("id" in column_names or "pk" in column_names) - and set(column_names) != {"id", "pk"} + and not set(column_names) == {"id", "pk"} ): - return next(c for c in column_names if c not in ("id", "pk")) + return [c for c in column_names if c not in ("id", "pk")][0] # Couldn't find a label: return None @@ -785,17 +755,6 @@ class Database: return hidden_tables - async def derived_table_dependencies(self): - """Return implementation tables and the tables they derive from.""" - schema_version = (await self.execute("PRAGMA schema_version")).first()[0] - if ( - self._cached_derived_table_dependencies is None - or self._cached_derived_table_dependencies[0] != schema_version - ): - dependencies = await self.execute_fn(sqlite_derived_table_dependencies) - self._cached_derived_table_dependencies = (schema_version, dependencies) - return self._cached_derived_table_dependencies[1] - async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] @@ -892,10 +851,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( "fn", - "isolated_connection", + "task_id", "loop", "reply_future", - "task_id", + "isolated_connection", "transaction", ) @@ -936,7 +895,7 @@ class QueryInterrupted(Exception): self.params = params def __str__(self): - return f"QueryInterrupted: {self.e}" + return "QueryInterrupted: {}".format(self.e) class MultipleValues(Exception): diff --git a/datasette/default_actions.py b/datasette/default_actions.py index ee165ae5..602e0df4 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -2,8 +2,8 @@ from datasette import hookimpl from datasette.permissions import Action from datasette.resources import ( DatabaseResource, - QueryResource, TableResource, + QueryResource, ) diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index 6def3698..f90a733e 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,17 +6,6 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType -_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) - - -def _normalize_http_url(value): - if not isinstance(value, str): - return None - normalized = value.strip() - if not _HTTP_URL_RE.fullmatch(normalized): - return None - return normalized - class UrlColumnType(ColumnType): name = "url" @@ -26,10 +15,7 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - normalized = _normalize_http_url(value) - if normalized is None: - return markupsafe.escape(value.strip()) - escaped = markupsafe.escape(normalized) + escaped = markupsafe.escape(value.strip()) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -37,7 +23,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if _normalize_http_url(value) is None: + if not re.match(r"^https?://\S+$", value.strip()): return "Invalid URL" return None diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index bff5f0a7..91c1c5aa 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -1,9 +1,8 @@ +from datasette import hookimpl import datetime import os import time -from datasette import hookimpl - def header(key, request): key = key.replace("_", "-").encode("utf-8") diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index dee5df42..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,29 +17,18 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -from .config import config_permissions_sql as config_permissions_sql -from .defaults import ( - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, -) -from .defaults import ( - default_action_permissions_sql as default_action_permissions_sql, -) -from .defaults import ( - # Avoid "datasette.default_permissions" does not explicitly export attribute - default_allow_sql_check as default_allow_sql_check, -) -from .defaults import ( - default_query_permissions_sql as default_query_permissions_sql, -) -from .restrictions import ( - ActorRestrictions as ActorRestrictions, -) - # Re-export all hooks and public utilities from .restrictions import ( actor_restrictions_sql as actor_restrictions_sql, -) -from .restrictions import ( restrictions_allow_action as restrictions_allow_action, + ActorRestrictions as ActorRestrictions, ) from .root import root_user_permissions_sql as root_user_permissions_sql +from .config import config_permissions_sql as config_permissions_sql +from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute + default_allow_sql_check as default_allow_sql_check, + default_action_permissions_sql as default_action_permissions_sql, + default_query_permissions_sql as default_query_permissions_sql, + DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, +) diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index a4f5a4de..aab87c1c 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration. from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple if TYPE_CHECKING: from datasette.app import Datasette @@ -55,8 +55,8 @@ class ConfigPermissionProcessor: def __init__( self, - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, ): self.datasette = datasette @@ -74,8 +74,8 @@ class ConfigPermissionProcessor: self.restrictions = actor.get("_r", {}) if actor else {} # Pre-compute restriction info for efficiency - self.restricted_databases: set[str] = set() - self.restricted_tables: set[tuple[str, str]] = set() + self.restricted_databases: Set[str] = set() + self.restricted_tables: Set[Tuple[str, str]] = set() if self.has_restrictions: self.restricted_databases = { @@ -92,27 +92,16 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - # Resolve identity keys once per action, rather than scanning the - # restriction allowlist for every configured table's allow block. - self.restricted_table_keys = { - (db, self.action_obj.normalize_child(table) if self.action_obj else table) - for db, table in self.restricted_tables - } - - def evaluate_allow_block(self, allow_block: Any) -> bool | None: + def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: """Evaluate an allow block against the current actor.""" if allow_block is None: return None - # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as - # integers, but should retain the CLI's boolean 1/0 behavior. - if isinstance(allow_block, int) and allow_block in (0, 1): - return bool(allow_block) return actor_matches_allow(self.actor, allow_block) def is_in_restriction_allowlist( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], ) -> bool: """Check if resource is allowed by actor restrictions.""" if not self.has_restrictions: @@ -132,10 +121,8 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - child_key = ( - self.action_obj.normalize_child(child) if self.action_obj else child - ) - if (parent, child_key) in self.restricted_table_keys: + table_actions = table_restrictions.get(child, []) + if self.action_checks.intersection(table_actions): return True else: # Parent query should proceed if any child in this database is allowlisted @@ -156,9 +143,9 @@ class ConfigPermissionProcessor: def add_permissions_rule( self, - parent: str | None, - child: str | None, - permissions_block: dict | None, + parent: Optional[str], + child: Optional[str], + permissions_block: Optional[dict], scope_desc: str, ) -> None: """Add a rule from a permissions:{action} block.""" @@ -178,8 +165,8 @@ class ConfigPermissionProcessor: def add_allow_block_rule( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], allow_block: Any, scope_desc: str, ) -> None: @@ -211,8 +198,8 @@ class ConfigPermissionProcessor: def _add_restriction_gate_denies( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], is_allowed: bool, scope_desc: str, ) -> None: @@ -244,7 +231,7 @@ class ConfigPermissionProcessor: if db_name == parent: self.collector.add(db_name, table_name, False, reason) - def process(self) -> PermissionSQL | None: + def process(self) -> Optional[PermissionSQL]: """Process all config rules and return combined PermissionSQL.""" self._process_root_permissions() self._process_databases() @@ -434,10 +421,10 @@ class ConfigPermissionProcessor: @hookimpl(specname="permission_resources_sql") async def config_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> list[PermissionSQL] | None: +) -> Optional[List[PermissionSQL]]: """ Apply permission rules from datasette.yaml configuration. diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 6f97812b..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -29,28 +29,29 @@ DEFAULT_ALLOW_ACTIONS = frozenset( @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: """ Enforce the default_allow_sql setting. When default_allow_sql is false (the default), execute-sql is denied unless explicitly allowed by config or other rules. """ - if action == "execute-sql" and not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") + if action == "execute-sql": + if not datasette.setting("default_allow_sql"): + return PermissionSQL.deny(reason="default_allow_sql is false") return None @hookimpl(specname="permission_resources_sql") async def default_action_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: """ Provide default allow rules for standard view/execute actions. @@ -70,10 +71,10 @@ async def default_action_permissions_sql( @hookimpl(specname="permission_resources_sql") async def default_query_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: actor_id = actor.get("id") if isinstance(actor, dict) else None if action not in {"view-query", "update-query", "delete-query"}: diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py index 5e59b7b4..47e03569 100644 --- a/datasette/default_permissions/helpers.py +++ b/datasette/default_permissions/helpers.py @@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Set if TYPE_CHECKING: from datasette.app import Datasette @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datasette.permissions import PermissionSQL -def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: +def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: """ Get all name variants for an action (full name and abbreviation). @@ -27,7 +27,7 @@ def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: return variants -def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool: +def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: """Check if an action (or its abbreviation) is in a list.""" return bool(get_action_name_variants(datasette, action).intersection(action_list)) @@ -36,8 +36,8 @@ def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool class PermissionRow: """A single permission rule row.""" - parent: str | None - child: str | None + parent: Optional[str] + child: Optional[str] allow: bool reason: str @@ -46,14 +46,14 @@ class PermissionRowCollector: """Collects permission rows and converts them to PermissionSQL.""" def __init__(self, prefix: str = "row"): - self.rows: list[PermissionRow] = [] + self.rows: List[PermissionRow] = [] self.prefix = prefix def add( self, - parent: str | None, - child: str | None, - allow: bool | None, + parent: Optional[str], + child: Optional[str], + allow: Optional[bool], reason: str, if_not_none: bool = False, ) -> None: @@ -62,7 +62,7 @@ class PermissionRowCollector: return self.rows.append(PermissionRow(parent, child, allow, reason)) - def to_permission_sql(self) -> PermissionSQL | None: + def to_permission_sql(self) -> Optional[PermissionSQL]: """Convert collected rows to a PermissionSQL object.""" if not self.rows: return None diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index d30ebd3f..a22cd7e5 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -8,7 +8,7 @@ contains allowlists of resources the actor can access. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Set, Tuple if TYPE_CHECKING: from datasette.app import Datasette @@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants class ActorRestrictions: """Parsed actor restrictions from the _r key.""" - global_actions: list[str] # _r.a - globally allowed actions + global_actions: List[str] # _r.a - globally allowed actions database_actions: dict # _r.d - {db_name: [actions]} table_actions: dict # _r.r - {db_name: {table: [actions]}} @classmethod - def from_actor(cls, actor: dict | None) -> ActorRestrictions | None: + def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: """Parse restrictions from actor dict. Returns None if no restrictions.""" if not actor: return None @@ -44,11 +44,11 @@ class ActorRestrictions: table_actions=restrictions.get("r", {}), ) - def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool: + def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: """Check if action is in the global allowlist.""" return action_in_list(datasette, action, self.global_actions) - def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]: + def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: """Get database names where this action is allowed.""" allowed = set() for db_name, db_actions in self.database_actions.items(): @@ -57,8 +57,8 @@ class ActorRestrictions: return allowed def get_allowed_tables( - self, datasette: Datasette, action: str - ) -> set[tuple[str, str]]: + self, datasette: "Datasette", action: str + ) -> Set[Tuple[str, str]]: """Get (database, table) pairs where this action is allowed.""" allowed = set() for db_name, tables in self.table_actions.items(): @@ -70,10 +70,10 @@ class ActorRestrictions: @hookimpl(specname="permission_resources_sql") async def actor_restrictions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> list[PermissionSQL] | None: +) -> Optional[List[PermissionSQL]]: """ Handle actor restriction-based permission rules. @@ -140,10 +140,10 @@ async def actor_restrictions_sql( def restrictions_allow_action( - datasette: Datasette, + datasette: "Datasette", restrictions: dict, action: str, - resource: str | tuple[str, str] | None, + resource: Optional[str | Tuple[str, str]], ) -> bool: """ Check if restrictions allow the requested action on the requested resource. @@ -185,15 +185,11 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - action_obj = datasette.actions.get(action) - normalize = action_obj.normalize_child if action_obj else lambda name: name - for table_name, table_allowed in ( - restrictions.get("r", {}).get(database, {}).items() - ): - if normalize(table_name) == normalize(table): - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py index 22d13f65..4931f7ff 100644 --- a/datasette/default_permissions/root.py +++ b/datasette/default_permissions/root.py @@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL @hookimpl(specname="permission_resources_sql") async def root_user_permissions_sql( - datasette: Datasette, - actor: dict | None, -) -> PermissionSQL | None: + datasette: "Datasette", + actor: Optional[dict], +) -> Optional[PermissionSQL]: """ Grant root user full permissions when --root flag is used. """ diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py deleted file mode 100644 index 11fd4008..00000000 --- a/datasette/default_permissions/sqlite_statistics.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Default table-access policy for SQLite optimizer statistics.""" - -import json - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - - -@hookimpl -def permission_resources_sql(action): - if action != "view-table": - return None - return PermissionSQL( - sql=""" - SELECT database_name AS parent, value AS child, 0 AS allow, - 'SQLite statistics tables are denied by default' AS reason - FROM catalog_databases - CROSS JOIN json_each(:sqlite_statistics_names) - """, - params={ - "sqlite_statistics_names": json.dumps( - ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] - ) - }, - ) diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py index 52daf8a2..7a359dc6 100644 --- a/datasette/default_permissions/tokens.py +++ b/datasette/default_permissions/tokens.py @@ -7,7 +7,7 @@ to datasette.verify_token() so all registered handlers are tried. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -17,13 +17,15 @@ from datasette.tokens import SignedTokenHandler @hookimpl -def register_token_handler(datasette: Datasette): +def register_token_handler(datasette: "Datasette"): """Register the default signed token handler.""" return SignedTokenHandler() @hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None: +async def actor_from_signed_api_token( + datasette: "Datasette", request +) -> Optional[dict]: """ Authenticate requests using API tokens by delegating to all registered token handlers via datasette.verify_token(). diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py index 0f2f32ef..e41434ef 100644 --- a/datasette/default_table_actions.py +++ b/datasette/default_table_actions.py @@ -20,7 +20,7 @@ def table_actions(datasette, actor, database, table, request): "label": "Alter table", "description": "Change columns and primary key for this table.", "attrs": { - "aria-label": f"Alter table {table}", + "aria-label": "Alter table {}".format(table), "data-table-action": "alter-table", }, } diff --git a/datasette/events.py b/datasette/events.py index 5f3fd06e..e8786da9 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -1,8 +1,7 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone - from datasette.hookspecs import hookimpl +from datetime import datetime, timezone @dataclass diff --git a/datasette/facets.py b/datasette/facets.py index 394eafad..abe0605e 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -1,13 +1,12 @@ import json import urllib - from datasette import hookimpl from datasette.database import QueryInterrupted from datasette.utils import ( - detect_json1, escape_sqlite, path_with_added_args, path_with_removed_args, + detect_json1, sqlite3, ) @@ -31,7 +30,7 @@ def load_facet_configs(request, table_config): assert ( len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, facet_config = next(iter(facet_config.items())) + type, facet_config = list(facet_config.items())[0] if isinstance(facet_config, str): facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( @@ -86,7 +85,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table - self.sql = sql or f"select * from {escape_sqlite(table)}" + self.sql = sql or f"select * from [{table}]" self.params = params or [] self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: @@ -161,13 +160,18 @@ class ColumnFacet(Facet): for column in columns: if column in already_enabled: continue - suggested_facet_sql = f""" - with limited as (select * from ({self.sql}) limit {self.suggest_consider}) - select {escape_sqlite(column)} as value, count(*) as n from limited + suggested_facet_sql = """ + with limited as (select * from ({sql}) limit {suggest_consider}) + select {column} as value, count(*) as n from limited where value is not null group by value - limit {facet_size + 1} - """ + limit {limit} + """.format( + column=escape_sqlite(column), + sql=self.sql, + limit=facet_size + 1, + suggest_consider=self.suggest_consider, + ) distinct_values = None try: distinct_values = await self.ds.execute( @@ -263,16 +267,11 @@ class ColumnFacet(Facet): for row in facet_rows: column_qs = column if column.startswith("_"): - column_qs = f"{column}__exact" - selected_args = { - key: str(row["value"]) - for key in (column_qs, f"{column}__exact") - if (key, str(row["value"])) in qs_pairs - } - selected = bool(selected_args) + column_qs = "{}__exact".format(column) + selected = (column_qs, str(row["value"])) in qs_pairs if selected: toggle_path = path_with_removed_args( - self.request, selected_args + self.request, {column_qs: str(row["value"])} ) else: toggle_path = path_with_added_args( @@ -343,12 +342,12 @@ class ArrayFacet(Facet): for v in await self.ds.execute( self.database, ( - f"select {escape_sqlite(column)} from ({self.sql}) " - f"where {escape_sqlite(column)} is not null " - f"and {escape_sqlite(column)} != '' " - f"and json_array_length({escape_sqlite(column)}) > 0 " + "select {column} from ({sql}) " + "where {column} is not null " + "and {column} != '' " + "and json_array_length({column}) > 0 " "limit 100" - ), + ).format(column=escape_sqlite(column), sql=self.sql), self.params, truncate=False, custom_time_limit=self.ds.setting( @@ -389,14 +388,14 @@ class ArrayFacet(Facet): source = source_and_config["source"] column = config.get("column") or config["simple"] # https://github.com/simonw/datasette/issues/448 - facet_sql = f""" - with inner as ({self.sql}), + facet_sql = """ + with inner as ({sql}), deduped_array_items as ( select distinct j.value, inner.* from - json_each([inner].{escape_sqlite(column)}) j + json_each([inner].{col}) j join inner ) select @@ -407,8 +406,12 @@ class ArrayFacet(Facet): group by value order by - count(*) desc, value limit {facet_size + 1} - """ + count(*) desc, value limit {limit} + """.format( + col=escape_sqlite(column), + sql=self.sql, + limit=facet_size + 1, + ) try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 0499d086..95cc5f37 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,12 +1,8 @@ -import json -import math -from typing import ClassVar - from datasette import hookimpl -from datasette.resources import DatabaseResource, TableResource -from datasette.utils.asgi import BadRequest +from datasette.resources import DatabaseResource from datasette.views.base import DatasetteError - +from datasette.utils.asgi import BadRequest +import json from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -52,20 +48,13 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which trusted fts_table to use. Query string parameters can - # repeat this mapping (for backwards compatibility), but must not select - # a different table or primary key. + # Figure out which fts_table to use table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = table_metadata.get("fts_table") + fts_table = request.args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = table_metadata.get("fts_pk", "rowid") - requested_fts_table = request.args.get("_fts_table") - requested_fts_pk = request.args.get("_fts_pk") - if (requested_fts_table and requested_fts_table != fts_table) or ( - requested_fts_pk and requested_fts_pk != fts_pk - ): - raise BadRequest("Invalid _fts_table or _fts_pk") + fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) search_args = { key: request.args[key] for key in request.args @@ -83,11 +72,6 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=fts_table), - actor=request.actor, - ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -115,9 +99,9 @@ def search_filters(request, database, table, datasette): fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), match_clause=( - f":search_{i}" + ":search_{}".format(i) if search_mode_raw - else f"escape_fts(:search_{i})" + else "escape_fts(:search_{})".format(i) ), ) ) @@ -148,18 +132,13 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=through_table), - actor=request.actor, - ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - fk_to_us = next( - (fk for fk in outgoing_foreign_keys if fk["other_table"] == table), - None, - ) - if fk_to_us is None: + try: + fk_to_us = [ + fk for fk in outgoing_foreign_keys if fk["other_table"] == table + ][0] + except IndexError: raise DatasetteError( "Invalid _through - could not find corresponding foreign key" ) @@ -203,17 +182,6 @@ class Filter: raise NotImplementedError -def _coerce_numeric_filter_value(value): - try: - return int(value) - except ValueError: - try: - converted = float(value) - except ValueError: - return value - return converted if math.isfinite(converted) else value - - class TemplatedFilter(Filter): def __init__( self, @@ -235,17 +203,13 @@ class TemplatedFilter(Filter): def where_clause(self, table, column, value, param_counter): converted = self.format.format(value) - if self.numeric: - converted = _coerce_numeric_filter_value(converted) + if self.numeric and converted.isdigit(): + converted = int(converted) if self.no_argument: - kwargs = {"c": _quote_sqlite_identifier(column)} + kwargs = {"c": column} converted = None else: - kwargs = { - "c": _quote_sqlite_identifier(column), - "p": f"p{param_counter}", - "t": _quote_sqlite_identifier(table), - } + kwargs = {"c": column, "p": f"p{param_counter}", "t": table} return self.sql_template.format(**kwargs), converted def human_clause(self, column, value): @@ -259,14 +223,6 @@ class TemplatedFilter(Filter): return template.format(c=column, v=value) -def _quote_sqlite_identifier(identifier): - # Preserve the historic always-quoted SQL generated by TemplatedFilter. - escaped = escape_sqlite(identifier) - if escaped == identifier: - return f'"{identifier}"' - return escaped - - class InFilter(Filter): key = "in" display = "in" @@ -308,56 +264,56 @@ class Filters: TemplatedFilter( "exact", "=", - "{c} = :{p}", + '"{c}" = :{p}', lambda c, v: "{c} = {v}" if v.isdigit() else '{c} = "{v}"', ), TemplatedFilter( "not", "!=", - "{c} != :{p}", + '"{c}" != :{p}', lambda c, v: "{c} != {v}" if v.isdigit() else '{c} != "{v}"', ), TemplatedFilter( "contains", "contains", - "{c} like :{p}", + '"{c}" like :{p}', '{c} contains "{v}"', format="%{}%", ), TemplatedFilter( "notcontains", "does not contain", - "{c} not like :{p}", + '"{c}" not like :{p}', '{c} does not contain "{v}"', format="%{}%", ), TemplatedFilter( "endswith", "ends with", - "{c} like :{p}", + '"{c}" like :{p}', '{c} ends with "{v}"', format="%{}", ), TemplatedFilter( "startswith", "starts with", - "{c} like :{p}", + '"{c}" like :{p}', '{c} starts with "{v}"', format="{}%", ), - TemplatedFilter("gt", ">", "{c} > :{p}", "{c} > {v}", numeric=True), + TemplatedFilter("gt", ">", '"{c}" > :{p}', "{c} > {v}", numeric=True), TemplatedFilter( - "gte", "\u2265", "{c} >= :{p}", "{c} \u2265 {v}", numeric=True + "gte", "\u2265", '"{c}" >= :{p}', "{c} \u2265 {v}", numeric=True ), - TemplatedFilter("lt", "<", "{c} < :{p}", "{c} < {v}", numeric=True), + TemplatedFilter("lt", "<", '"{c}" < :{p}', "{c} < {v}", numeric=True), TemplatedFilter( - "lte", "\u2264", "{c} <= :{p}", "{c} \u2264 {v}", numeric=True + "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True ), - TemplatedFilter("like", "like", "{c} like :{p}", '{c} like "{v}"'), + TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'), TemplatedFilter( - "notlike", "not like", "{c} not like :{p}", '{c} not like "{v}"' + "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"' ), - TemplatedFilter("glob", "glob", "{c} glob :{p}", '{c} glob "{v}"'), + TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'), InFilter(), NotInFilter(), ] @@ -366,13 +322,13 @@ class Filters: TemplatedFilter( "arraycontains", "array contains", - """:{p} in (select value from json_each({t}.{c}))""", + """:{p} in (select value from json_each([{t}].[{c}]))""", '{c} contains "{v}"', ), TemplatedFilter( "arraynotcontains", "array does not contain", - """:{p} not in (select value from json_each({t}.{c}))""", + """:{p} not in (select value from json_each([{t}].[{c}]))""", '{c} does not contain "{v}"', ), ] @@ -380,34 +336,36 @@ class Filters: else [] ) + [ - TemplatedFilter("date", "date", "date({c}) = :{p}", '"{c}" is on date {v}'), TemplatedFilter( - "isnull", "is null", "{c} is null", "{c} is null", no_argument=True + "date", "date", 'date("{c}") = :{p}', '"{c}" is on date {v}' + ), + TemplatedFilter( + "isnull", "is null", '"{c}" is null', "{c} is null", no_argument=True ), TemplatedFilter( "notnull", "is not null", - "{c} is not null", + '"{c}" is not null', "{c} is not null", no_argument=True, ), TemplatedFilter( "isblank", "is blank", - "({c} is null or {c} = '')", + '("{c}" is null or "{c}" = "")', "{c} is blank", no_argument=True, ), TemplatedFilter( "notblank", "is not blank", - "({c} is not null and {c} != '')", + '("{c}" is not null and "{c}" != "")', "{c} is not blank", no_argument=True, ), ] ) - _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters} + _filters_by_key = {f.key: f for f in _filters} def __init__(self, pairs): self.pairs = pairs diff --git a/datasette/fixtures.py b/datasette/fixtures.py index 049e35ed..7c85e16a 100644 --- a/datasette/fixtures.py +++ b/datasette/fixtures.py @@ -1,10 +1,9 @@ +from datasette.utils.sqlite import sqlite3 +from datasette.utils import documented import itertools import random import string -from datasette.utils import documented -from datasette.utils.sqlite import sqlite3 - __all__ = [ "EXTRA_DATABASE_SQL", "TABLES", @@ -347,7 +346,9 @@ CREATE VIEW searchable_view_configured_by_metadata AS + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + "\n".join( [ - f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");' + 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + a=a, b=b, c=c, content=content + ) for a, b, c, content in generate_compound_rows(1001) ] ) diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 67bf0d8b..91b1ff96 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,5 +1,4 @@ -from datasette import Response, hookimpl - +from datasette import hookimpl, Response from .utils import add_cors_headers diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index ef6c7b7e..e255ddf2 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,21 +1,16 @@ -import traceback - -from markupsafe import Markup - -from datasette import Response, hookimpl - +from datasette import hookimpl, Response from .utils import add_cors_headers, error_body from .utils.asgi import ( Base400, ) from .views.base import DatasetteError +from markupsafe import Markup +import traceback -# Debugger imports are deliberate - they back the "pdb" setting, which drops -# into a debugger on unhandled exceptions try: - import ipdb as pdb # noqa: T100 + import ipdb as pdb except ImportError: - import pdb # noqa: T100 + import pdb try: import rich @@ -59,10 +54,6 @@ def handle_exception(datasette, request, exception): body = dict(info) body.update(error_body(plain_message or message, status)) return Response.json(body, status=status, headers=headers) - if request.path.split("?")[0].endswith(".csv"): - return Response.text( - plain_message or message, status=status, headers=headers - ) info.update( { "ok": False, @@ -78,7 +69,7 @@ def handle_exception(datasette, request, exception): dict( info, urls=datasette.urls, - menu_links=list, + menu_links=lambda: [], ) ), status=status, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 49d8e8ea..7c56f882 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -1,4 +1,5 @@ -from pluggy import HookimplMarker, HookspecMarker +from pluggy import HookimplMarker +from pluggy import HookspecMarker hookspec = HookspecMarker("datasette") hookimpl = HookimplMarker("datasette") @@ -9,11 +10,6 @@ def startup(datasette): """Fires directly after Datasette first starts running""" -@hookspec -def shutdown(datasette): - """Called once when the Datasette server is shutting down""" - - @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" @@ -50,7 +46,7 @@ def extra_body_script( def extra_template_vars( template, database, table, columns, view_name, request, datasette ): - """Extra template variables to be made available to the template - can return dict, None, callable or awaitable""" + """Extra template variables to be made available to the template - can return dict or callable or awaitable""" @hookspec diff --git a/datasette/inspect.py b/datasette/inspect.py index b126ce5c..5e681e03 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -1,13 +1,13 @@ import hashlib from .utils import ( + detect_spatialite, detect_fts, detect_primary_keys, - detect_spatialite, escape_sqlite, get_all_foreign_keys, - sqlite3, table_columns, + sqlite3, ) HASH_BLOCK_SIZE = 1024 * 1024 @@ -95,10 +95,10 @@ def inspect_tables(conn, database_metadata): """) ] - for t, table_info in tables.items(): + for t in tables.keys(): for hidden_table in hidden_tables: if t == hidden_table or t.startswith(hidden_table): - table_info["hidden"] = True + tables[t]["hidden"] = True continue return tables diff --git a/datasette/jump.py b/datasette/jump.py index d70d33df..d138e827 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -21,7 +21,7 @@ class JumpSQL: search_text: str | None = None, display_name: str | None = None, item_type: str = "menu", - ) -> JumpSQL: + ) -> "JumpSQL": if search_text is None: search_text = " ".join( text for text in (label, display_name, description) if text is not None diff --git a/datasette/permissions.py b/datasette/permissions.py index 2d242560..786dc026 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,11 +1,7 @@ -import contextvars from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple - -_SQLITE_IDENTIFIER_CASE = str.maketrans( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" -) +import contextvars # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( @@ -53,15 +49,6 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables - case_insensitive_child: bool = False - - @classmethod - def normalize_child(cls, child: str | None) -> str | None: - """Return a comparison key without changing the resource's display name.""" - if cls.case_insensitive_child and child is not None: - # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. - return child.translate(_SQLITE_IDENTIFIER_CASE) - return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -85,8 +72,8 @@ class Resource(ABC): ) def __repr__(self) -> str: - return ( - f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})" + return "{}(parent={!r}, child={!r})".format( + self.__class__.__name__, self.parent, self.child ) @property @@ -142,6 +129,7 @@ class Resource(ABC): Must return two columns: parent, child """ + pass class AllowedResource(NamedTuple): @@ -159,11 +147,6 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed - def normalize_child(self, child: str | None) -> str | None: - if self.resource_class is None: - return child - return self.resource_class.normalize_child(child) - @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 6a4d7da7..ae2cb17d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,14 +1,20 @@ import importlib -import importlib.metadata as importlib_metadata -import importlib.resources as importlib_resources import os -import sys -from pprint import pprint - import pluggy - +from pprint import pprint +import sys from . import hookspecs +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -18,7 +24,6 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", - "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", @@ -80,7 +85,7 @@ if DATASETTE_LOAD_PLUGINS is not None: # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) except importlib_metadata.PackageNotFoundError: - sys.stderr.write(f"Plugin {package_name} could not be found\n") + sys.stderr.write("Plugin {} could not be found\n".format(package_name)) # Load default plugins diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 9ace865b..63d22fe8 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,17 +1,15 @@ +from datasette import hookimpl +import click import json import os import re from subprocess import CalledProcessError, check_call, check_output -import click - -from datasette import hookimpl - -from ..utils import temporary_docker_directory from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) +from ..utils import temporary_docker_directory @hookimpl @@ -221,7 +219,7 @@ def publish_subcommand(publish): check_call( "gcloud builds submit --tag {}{}".format( - image_id, f" --timeout {timeout}" if timeout else "" + image_id, " --timeout {}".format(timeout) if timeout else "" ), shell=True, ) @@ -233,7 +231,7 @@ def publish_subcommand(publish): ("--min-instances", min_instances), ): if value is not None: - extra_deploy_options.append(f"{option} {value}") + extra_deploy_options.append("{} {}".format(option, value)) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, @@ -260,16 +258,24 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi ) from exc describe_cmd = ( - f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} " - f"--location {artifact_region} --quiet" + "gcloud artifacts repositories describe {repo} --project {project} " + "--location {location} --quiet" + ).format( + repo=artifact_repository, + project=artifact_project, + location=artifact_region, ) try: check_call(describe_cmd, shell=True) return except CalledProcessError: create_cmd = ( - f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker " - f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet' + "gcloud artifacts repositories create {repo} --repository-format=docker " + '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' + ).format( + repo=artifact_repository, + location=artifact_region, + project=artifact_project, ) try: check_call(create_cmd, shell=True) diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 27dfd4bf..29665eb3 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -1,11 +1,9 @@ +from ..utils import StaticMount +import click import os import shutil import sys -import click - -from ..utils import StaticMount - def add_common_publish_arguments_and_options(subcommand): for decorator in reversed( @@ -78,7 +76,9 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): """Exit (with error message) if ``binary` isn't installed""" if not shutil.which(binary): click.secho( - f"Publishing to {publish_target} requires {binary} to be installed and configured", + "Publishing to {publish_target} requires {binary} to be installed and configured".format( + publish_target=publish_target, binary=binary + ), bg="red", fg="white", bold=True, diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index b0290833..f576a346 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -1,21 +1,19 @@ +from contextlib import contextmanager +from datasette import hookimpl +import click import json import os import pathlib import shlex import shutil -import tempfile -from contextlib import contextmanager from subprocess import call, check_output - -import click - -from datasette import hookimpl -from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata +import tempfile from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) +from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata @hookimpl @@ -236,7 +234,7 @@ def temporary_heroku_directory( extras.extend(["--static", f"{mount_point}:{mount_point}"]) quoted_files = " ".join( - [f"-i {shlex.quote(file_name)}" for file_name in file_names] + ["-i {}".format(shlex.quote(file_name)) for file_name in file_names] ) procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) diff --git a/datasette/renderer.py b/datasette/renderer.py index 0e01f52f..7c94f6ee 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,13 +1,12 @@ import json - from datasette.extras import extra_names_from_request from datasette.utils import ( - CustomJSONEncoder, error_body, - path_from_row_pks, - remove_infinites, - sqlite3, value_as_boolean, + remove_infinites, + CustomJSONEncoder, + path_from_row_pks, + sqlite3, ) from datasette.utils.asgi import Response diff --git a/datasette/resources.py b/datasette/resources.py index 29bf7b1e..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,7 +25,6 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource - case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/app.css b/datasette/static/app.css index 0297371c..79fb1a73 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1,144 +1,3 @@ -/* Shared modal styles. */ -datasette-modal { - display: contents; -} - -dialog.datasette-modal { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; - max-height: calc(100dvh - 32px); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.datasette-modal[open] { - display: flex; - flex-direction: column; -} - -dialog.datasette-modal::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -@keyframes datasette-modal-slide-in { - from { opacity: 0; transform: translateY(-20px) scale(0.95); } - to { opacity: 1; transform: translateY(0) scale(1); } -} - -@keyframes datasette-modal-fade-in { - from { opacity: 0; } - to { opacity: 1; } -} - -:where(.datasette-modal) .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 12px; - flex-shrink: 0; - min-width: 0; -} - -:where(.datasette-modal) .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -:where(.datasette-modal) .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -:where(.datasette-modal) .modal-body { - min-height: 0; - overflow: auto; - padding: 16px 24px 24px; -} - -:where(.datasette-modal) .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -:where(.datasette-modal) .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -:where(.datasette-modal) .modal-btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -:where(.datasette-modal) .modal-btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -:where(.datasette-modal) .modal-btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -:where(.datasette-modal) .modal-btn-primary { - background: var(--accent); - color: #fff; -} - -:where(.datasette-modal) .modal-btn-primary:hover { - background: #1949b8; -} - -:where(.datasette-modal) .modal-btn:disabled { - opacity: 0.65; - cursor: wait; -} - -@media (prefers-reduced-motion: reduce) { - dialog.datasette-modal, - dialog.datasette-modal::backdrop { - animation: none; - } -} - /* Reset and Page Setup ==================================================== */ /* Reset from http://meyerweb.com/eric/tools/css/reset/ @@ -204,7 +63,7 @@ em { } /* end reset */ -/* Shared modal CSS variables */ +/* Modal CSS variables (shared by web components via Shadow DOM) */ :root { --modal-backdrop-bg: rgba(0, 0, 0, 0.5); --modal-backdrop-blur: blur(4px); @@ -357,49 +216,6 @@ a:active { text-decoration: underline; } -.table-summary .count-all ~ .table-summary-description { - margin-left: 0.5rem; -} - -.table-summary .count-error:not(:empty) { - display: block; - margin-top: 0.25rem; - font-size: 0.875rem; - font-weight: 400; - line-height: 1.5; -} - -button.count-all { - background: none; - border: none; - padding: 3px 0; - margin-left: 0.25rem; - color: #276890; - font-family: inherit; - font-size: 0.8125rem; - font-weight: 400; - line-height: 1.5; - cursor: pointer; -} - -button.count-all:hover, -button.count-all:focus-visible { - text-decoration: underline; -} - -button.count-all:disabled { - color: #596478; - cursor: wait; -} - -@media (pointer: coarse) { - button.count-all { - min-height: 44px; - padding-left: 7px; - padding-right: 7px; - } -} - button.button-as-link { background: none; border: none; @@ -681,7 +497,10 @@ table.rows-and-columns td em { color: #aaa; } table.rows-and-columns th { + background: #F8FAFB; padding-right: 1em; + position: relative; + z-index: 1; } table.rows-and-columns a:link { text-decoration: none; @@ -1122,552 +941,84 @@ p.zero-results { display: none; } -/* navigation-search */ -navigation-search { - display: contents; -} - -navigation-search dialog.datasette-modal { - max-width: 90vw; - width: 600px; - max-height: 80vh; -} - -navigation-search .search-container { - display: flex; - flex-direction: column; -} - -navigation-search .search-input-wrapper { - padding: 1.25rem; - border-bottom: 1px solid #e5e7eb; - display: flex; - gap: 0.5rem; - align-items: center; -} - -navigation-search .search-input { - width: 100%; - flex: 1; - min-width: 0; - padding: 0.75rem 1rem; - font-size: 1rem; - border: 2px solid #e5e7eb; - border-radius: 0.5rem; - outline: none; - transition: border-color 0.2s; - box-sizing: border-box; -} - -navigation-search .search-input:focus { - border-color: #2563eb; -} - -navigation-search .close-search { - background: transparent; - border: 1px solid transparent; - border-radius: 0.375rem; - color: #4b5563; - cursor: pointer; - flex: 0 0 auto; - font: inherit; - font-size: 1.5rem; - height: 2.75rem; - line-height: 1; - width: 2.75rem; -} - -navigation-search .close-search:hover, -navigation-search .close-search:focus { - background-color: #f3f4f6; - border-color: #d1d5db; -} - -navigation-search .results-container { - box-sizing: content-box; - height: calc(80vh - 180px); - padding: 0.5rem; -} - -navigation-search .results-list:empty { - display: none; -} - -navigation-search .result-item { - padding: 0.875rem 1rem; - cursor: pointer; - border-radius: 0.5rem; - transition: background-color 0.15s; - display: flex; - align-items: center; - gap: 0.75rem; -} - -navigation-search .result-item:hover { - background-color: #f3f4f6; -} - -navigation-search .result-item.selected { - background-color: #dbeafe; -} - -navigation-search .result-item > div { - flex: 1; - min-width: 0; -} - -navigation-search .jump-start-content { - border-bottom: 1px solid #e5e7eb; - margin-bottom: 0.5rem; - padding: 0.5rem 0.5rem 1rem; -} - -navigation-search .jump-start-content:empty { - display: none; -} - -navigation-search .result-name { - font-weight: 500; - color: #111827; -} - -navigation-search .result-label { - font-size: 0.875rem; - color: #4b5563; -} - -navigation-search .result-type { - color: #4b5563; - font-size: 0.75rem; - font-weight: 600; - text-transform: uppercase; -} - -navigation-search .result-url { - font-size: 0.875rem; - color: #6b7280; -} - -navigation-search .result-description { - color: #374151; - display: -webkit-box; - font-size: 0.8125rem; - line-height: 1.35; - margin-top: 0.35rem; - overflow: hidden; - -webkit-box-orient: vertical; - -webkit-line-clamp: 2; -} - -navigation-search .results-heading { - color: #4b5563; - font-size: 0.75rem; - font-weight: 600; - letter-spacing: 0; - padding: 0.5rem 1rem 0.25rem; - text-transform: uppercase; -} - -navigation-search .recent-actions { - padding: 0.25rem 1rem 0.75rem; -} - -navigation-search .clear-recent { - background: transparent; - border: 0; - color: #2563eb; - cursor: pointer; - font: inherit; - font-size: 0.875rem; - padding: 0; -} - -navigation-search .clear-recent:hover { - text-decoration: underline; -} - -navigation-search .no-results { - padding: 2rem; - text-align: center; - color: #6b7280; -} - -navigation-search .hint-text { - padding: 0.75rem 1.25rem; - font-size: 0.875rem; - color: #6b7280; - border-top: 1px solid #e5e7eb; - display: flex; - gap: 1rem; - flex-wrap: wrap; -} - -navigation-search .hint-text kbd { - background: #f3f4f6; - padding: 0.125rem 0.375rem; - border-radius: 0.25rem; - font-size: 0.75rem; - border: 1px solid #d1d5db; - font-family: monospace; -} - -navigation-search .visually-hidden { - border: 0; - clip: rect(0 0 0 0); - height: 1px; - margin: -1px; - overflow: hidden; - padding: 0; - position: absolute; - white-space: nowrap; - width: 1px; -} - -@media (max-width: 640px) { - navigation-search dialog.datasette-modal { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; +@keyframes datasette-modal-slide-in { + from { + opacity: 0; + transform: translateY(-20px) scale(0.95); } - - navigation-search .search-input-wrapper { - padding: 1rem; - } - - navigation-search .search-input { - font-size: 16px; - } - - navigation-search .result-item { - padding: 1rem 0.75rem; - } - - navigation-search .hint-text { - font-size: 0.8rem; - padding: 0.5rem 1rem; + to { + opacity: 1; + transform: translateY(0) scale(1); } } +@keyframes datasette-modal-fade-in { + from { opacity: 0; } + to { opacity: 1; } +} -/* column-chooser */ -column-chooser { - display: contents; +dialog.mobile-column-actions-dialog { --ink: #0f0f0f; --paper: #eef6ff; --muted: #6b6b6b; --rule: #d8e6f5; --accent: #1a56db; - --accent-light: #e8effd; --card: #ffffff; -} - -column-chooser * { - box-sizing: border-box; - margin: 0; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); padding: 0; -} - -column-chooser dialog.datasette-modal { - width: 100%; - max-width: 420px; - max-height: min(640px, calc(100vh - 32px)); - -webkit-user-select: none; - -webkit-touch-callout: none; - -webkit-tap-highlight-color: transparent; -} - -column-chooser dialog.datasette-modal[open] { - height: min(640px, calc(100vh - 32px)); -} - -column-chooser .modal-header { - padding: 20px 24px 16px; - justify-content: space-between; -} - -column-chooser .list-toolbar { - padding: 6px 24px; - border-bottom: 1px solid var(--rule); - display: flex; - gap: 12px; - flex-shrink: 0; -} - -column-chooser .list-toolbar button { - background: var(--accent-light); - border: 1px solid var(--rule); - border-radius: 4px; - font-family: inherit; - font-size: 0.75rem; - color: var(--accent); - cursor: pointer; - padding: 3px 10px; - transition: - background 0.12s, - color 0.12s; -} - -column-chooser .list-toolbar button:hover { - background: var(--accent); - color: white; -} - -column-chooser .list-wrap { - flex: 1; - padding: 0; - overflow-x: hidden; - position: relative; - overscroll-behavior: contain; - -webkit-overflow-scrolling: touch; -} - -column-chooser .list-wrap::before, -column-chooser .list-wrap::after { - content: ""; - position: sticky; - display: block; - left: 0; - right: 0; - height: 20px; - pointer-events: none; - z-index: 5; - transition: opacity 0.2s; -} - -column-chooser .list-wrap::before { - top: 0; - background: linear-gradient( - to bottom, - rgba(255, 255, 255, 0.9), - transparent - ); -} - -column-chooser .list-wrap::after { - bottom: 0; - background: linear-gradient(to top, rgba(255, 255, 255, 0.9), transparent); - margin-top: -20px; -} - -column-chooser .scroll-zone { - position: absolute; - left: 0; - right: 0; - height: 72px; - pointer-events: none; - z-index: 10; -} - -column-chooser .scroll-zone-top { - top: 0; -} - -column-chooser .scroll-zone-bot { - bottom: 0; -} - -column-chooser .drag-list { - list-style: none; - padding: 4px 0; -} - -column-chooser .drag-item { - display: flex; - align-items: center; - background: white; - border-bottom: 1px solid var(--rule); - user-select: none; - -webkit-user-select: none; - -webkit-touch-callout: none; - position: relative; - transition: background 0.08s; -} - -column-chooser .drag-item:last-child { - border-bottom: none; -} - -column-chooser .drag-handle { - display: flex; - align-items: center; - justify-content: center; - width: 48px; - height: 48px; - flex-shrink: 0; - cursor: grab; - color: #c8c4bc; - touch-action: none; - transition: color 0.15s; -} - -column-chooser .drag-handle:hover { - color: var(--accent); -} - -column-chooser .drag-handle svg { - pointer-events: none; - display: block; -} - -column-chooser .drag-item-content { - display: flex; - align-items: center; - flex: 1; - min-width: 0; - cursor: pointer; -} - -column-chooser .drag-item-check { - display: flex; - align-items: center; - width: 32px; - height: 48px; - flex-shrink: 0; -} - -column-chooser .drag-item-check input[type="checkbox"] { - width: 16px; - height: 16px; - accent-color: var(--accent); - cursor: pointer; -} - -column-chooser .drag-item-label { - flex: 1; - font-size: 0.9rem; - line-height: 48px; - padding-right: 16px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - cursor: default; -} - -column-chooser .drag-item.is-dragging { - opacity: 0; -} - -column-chooser .drop-indicator { - position: absolute; - left: 48px; - right: 0; - height: 2px; - background: var(--accent); - border-radius: 99px; - pointer-events: none; - z-index: 20; - display: none; -} - -column-chooser .drop-indicator.top { - top: -1px; - display: block; -} - -column-chooser .drop-indicator.bottom { - bottom: -1px; - display: block; -} - -column-chooser .drag-ghost { - position: fixed; - pointer-events: none; - z-index: 9999; - background: white; - border-radius: 6px; - box-shadow: - 0 8px 32px rgba(0, 0, 0, 0.18), - 0 2px 8px rgba(0, 0, 0, 0.1); - display: flex; - align-items: center; - border: 1.5px solid var(--accent-light); - opacity: 0.97; - will-change: transform; - font-family: - system-ui, - -apple-system, - sans-serif; -} - -column-chooser .scroll-pulse { - position: absolute; - left: 50%; - transform: translateX(-50%); - width: 32px; - height: 32px; - border-radius: 50%; - background: var(--accent); - opacity: 0; - pointer-events: none; - z-index: 10; - transition: opacity 0.15s; -} - -column-chooser .scroll-pulse.top { - top: 8px; -} - -column-chooser .scroll-pulse.bot { - bottom: 8px; -} - -column-chooser .scroll-pulse.active { - opacity: 0.18; - animation: column-chooser-pulse 0.8s ease-in-out infinite; -} - -@keyframes column-chooser-pulse { - 0%, - 100% { - transform: translateX(-50%) scale(1); - opacity: 0.18; - } - 50% { - transform: translateX(-50%) scale(1.5); - opacity: 0.07; - } -} - -column-chooser .modal-btn-primary { - color: white; -} - -column-chooser .modal-btn-primary:hover { - background: #1448c0; -} - -column-chooser .list-wrap::-webkit-scrollbar { - width: 5px; -} - -column-chooser .list-wrap::-webkit-scrollbar-track { - background: transparent; -} - -column-chooser .list-wrap::-webkit-scrollbar-thumb { - background: var(--rule); - border-radius: 99px; -} - -column-chooser input, -column-chooser textarea { - -webkit-user-select: auto; - user-select: auto; -} - -dialog.mobile-column-actions-dialog { + margin: auto; width: min(420px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(640px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.mobile-column-actions-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.mobile-column-actions-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .mobile-column-actions-dialog .modal-header { padding: 20px 24px 16px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.mobile-column-actions-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.mobile-column-actions-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; } .mobile-column-actions-dialog .list-wrap { flex: 1 1 auto; - padding: 0; + min-height: 0; + overflow-y: auto; overflow-x: hidden; position: relative; overscroll-behavior: contain; @@ -1794,12 +1145,102 @@ dialog.mobile-column-actions-dialog { font-size: 0.85em; } +.mobile-column-actions-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.mobile-column-actions-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.mobile-column-actions-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.mobile-column-actions-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.mobile-column-actions-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + dialog.set-column-type-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(520px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(720px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.set-column-type-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.set-column-type-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .set-column-type-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.set-column-type-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.set-column-type-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; } .set-column-type-status, @@ -1821,6 +1262,8 @@ dialog.set-column-type-dialog { } .set-column-type-options { + padding: 16px 24px 24px; + overflow-y: auto; display: grid; gap: 12px; } @@ -1862,6 +1305,60 @@ dialog.set-column-type-dialog { font-size: 0.9rem; } +.set-column-type-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.set-column-type-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.set-column-type-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.set-column-type-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.set-column-type-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.set-column-type-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.set-column-type-dialog .btn-primary:hover { + background: #1949b8; +} + +.set-column-type-dialog .btn:disabled { + opacity: 0.65; + cursor: wait; +} + .row-mutation-status { margin: 0 0 0.75rem; padding: 8px 10px; @@ -1895,11 +1392,46 @@ button.table-insert-row svg { } dialog.row-delete-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(440px, calc(100vw - 32px)); + max-width: 95vw; + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.row-delete-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.row-delete-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; } .row-delete-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; justify-content: flex-start; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .row-delete-dialog .modal-title { @@ -1908,6 +1440,9 @@ dialog.row-delete-dialog { gap: 0.35rem; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .row-delete-message, @@ -1939,12 +1474,94 @@ dialog.row-delete-dialog { .row-delete-dialog .modal-footer { padding: 18px 20px 14px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); margin-top: 18px; } +.row-delete-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.row-delete-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.row-delete-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.row-delete-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.row-delete-dialog .btn-primary:hover { + background: #1949b8; +} + +.row-delete-dialog .btn:disabled { + opacity: 0.65; + cursor: wait; +} + dialog.row-edit-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(720px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.row-edit-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.row-edit-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.row-edit-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .row-edit-dialog .modal-title { @@ -1953,6 +1570,9 @@ dialog.row-edit-dialog { gap: 0.35rem; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .row-edit-dialog .modal-title .row-dialog-action, @@ -2020,6 +1640,8 @@ dialog.row-edit-dialog { .row-edit-fields { display: grid; gap: 14px; + padding: 16px 24px 24px; + overflow-y: auto; } .row-edit-fields[hidden], @@ -2299,6 +1921,8 @@ textarea.row-edit-input { .row-edit-bulk { display: grid; gap: 8px; + padding: 16px 24px 24px; + overflow-y: auto; } .row-edit-bulk-editor { @@ -2318,7 +1942,7 @@ textarea.row-edit-input { justify-content: flex-start; } -.row-edit-bulk-actions .modal-btn { +.row-edit-bulk-actions .btn { padding-left: 12px; padding-right: 12px; } @@ -2562,6 +2186,17 @@ datasette-autocomplete input[type="text"], max-width: 46rem; } +.row-edit-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + .row-edit-mode-link { color: var(--accent); font-size: 0.9rem; @@ -2572,14 +2207,84 @@ datasette-autocomplete input[type="text"], display: none; } -.row-edit-dialog .modal-btn:disabled { +.row-edit-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.row-edit-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.row-edit-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.row-edit-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.row-edit-dialog .btn-primary:hover { + background: #1949b8; +} + +.row-edit-dialog .btn:disabled { opacity: 0.55; cursor: not-allowed; } dialog.table-create-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(980px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-create-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-create-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-create-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .table-create-dialog .modal-title { @@ -2587,6 +2292,9 @@ dialog.table-create-dialog { align-items: center; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .table-create-form { @@ -2614,6 +2322,8 @@ dialog.table-create-dialog { .table-create-fields { display: grid; gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; } .table-create-field { @@ -3023,6 +2733,17 @@ select.table-create-input { outline-offset: 1px; } +.table-create-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + .table-create-mode-link { color: var(--accent); font-size: 0.9rem; @@ -3033,7 +2754,39 @@ select.table-create-input { display: none; } -.table-create-dialog .modal-btn:disabled, +.table-create-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-create-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-create-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-create-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-create-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-create-dialog .btn:disabled, .table-create-add-column:disabled, .table-create-icon-button:disabled { opacity: 0.55; @@ -3041,8 +2794,46 @@ select.table-create-input { } dialog.table-alter-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; width: min(980px, calc(100vw - 32px)); + max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-alter-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-alter-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-alter-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; } .table-alter-dialog .modal-title { @@ -3050,6 +2841,9 @@ dialog.table-alter-dialog { align-items: center; min-width: 0; max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); } .table-alter-form { @@ -3077,6 +2871,8 @@ dialog.table-alter-dialog { .table-alter-fields { display: grid; gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; } .table-alter-table-options { @@ -3110,6 +2906,8 @@ dialog.table-alter-dialog { .table-alter-review { display: grid; gap: 12px; + overflow-y: auto; + padding: 16px 24px 24px; } .table-alter-review[hidden] { @@ -3403,29 +3201,72 @@ select.table-alter-input { outline-offset: 1px; } -.table-alter-dialog .modal-btn-danger { +.table-alter-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.table-alter-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-alter-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-alter-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-alter-dialog .btn-danger { background: #b91c1c; color: #fff; margin-right: auto; } -.table-alter-dialog .modal-btn-danger:hover { +.table-alter-dialog .btn-danger:hover { background: #991b1b; } -.table-alter-dialog .modal-btn-danger:disabled, -.table-alter-dialog .modal-btn-danger:disabled:hover { +.table-alter-dialog .btn-danger:disabled, +.table-alter-dialog .btn-danger:disabled:hover { background: #d98c8c; color: #fff; } -.table-alter-dialog .modal-btn-primary:disabled, -.table-alter-dialog .modal-btn-primary:disabled:hover { +.table-alter-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-alter-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-alter-dialog .btn-primary:disabled, +.table-alter-dialog .btn-primary:disabled:hover { background: #a0aec0; color: #fff; } -.table-alter-dialog .modal-btn:disabled, +.table-alter-dialog .btn:disabled, .table-alter-add-column:disabled, .table-alter-icon-button:disabled { opacity: 0.55; diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index 29729f27..198641f3 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -1,9 +1,7 @@ -let columnChooserInstanceCounter = 0; - class ColumnChooser extends HTMLElement { constructor() { super(); - this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`; + this.attachShadow({ mode: "open" }); // State this._items = []; @@ -28,60 +26,375 @@ class ColumnChooser extends HTMLElement { // Bound handlers this._onMove = this._onMove.bind(this); this._onUp = this._onUp.bind(this); - } - connectedCallback() { - if (this._modal) return; - this.innerHTML = ` - + this.shadowRoot.innerHTML = ` + + +
- Choose columns - + Choose columns +
- - + +
-
-
-
-
    +
    +
    +
    +
      - - - + + +
      -
      + `; // DOM refs - this._modal = this.querySelector("datasette-modal"); - this._listWrap = this.querySelector(".list-wrap"); - this._dragList = this.querySelector(".drag-list"); - this._pulseTop = this.querySelector(".scroll-pulse.top"); - this._pulseBot = this.querySelector(".scroll-pulse.bot"); - this._selectAllBtn = this.querySelector(".select-all"); - this._deselectAllBtn = this.querySelector(".deselect-all"); - this._cancelBtn = this.querySelector(".modal-btn-ghost"); - this._applyBtn = this.querySelector(".modal-btn-primary"); - this._countEl = this.querySelector(".modal-meta"); - this._footerEl = this.querySelector(".footer-info"); + this._dialog = this.shadowRoot.querySelector("dialog"); + this._listWrap = this.shadowRoot.getElementById("listWrap"); + this._dragList = this.shadowRoot.getElementById("dragList"); + this._pulseTop = this.shadowRoot.getElementById("pulseTop"); + this._pulseBot = this.shadowRoot.getElementById("pulseBot"); + this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn"); + this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn"); + this._cancelBtn = this.shadowRoot.getElementById("cancelBtn"); + this._applyBtn = this.shadowRoot.getElementById("applyBtn"); + this._countEl = this.shadowRoot.getElementById("selectedCount"); + this._footerEl = this.shadowRoot.getElementById("footerInfo"); // Event listeners this._selectAllBtn.addEventListener("click", () => this._selectAll()); this._deselectAllBtn.addEventListener("click", () => this._deselectAll()); - this._cancelBtn.addEventListener("click", () => - this._modal.requestClose("cancel"), - ); + this._cancelBtn.addEventListener("click", () => this._close()); this._applyBtn.addEventListener("click", () => this._apply()); - this._modal.beforeClose = () => { - this._items = this._savedItems ? [...this._savedItems] : this._items; - this._checked = this._savedChecked - ? new Set(this._savedChecked) - : this._checked; - return true; - }; + this._dialog.addEventListener("click", (e) => { + if (e.target === this._dialog) this._close(); + }); + this._dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this._close(); + }); } /** @@ -101,11 +414,19 @@ class ColumnChooser extends HTMLElement { this._savedChecked = new Set(this._checked); this._render(); - this._modal.show(); + this._dialog.showModal(); } // ── Internal methods ── + _close() { + this._items = this._savedItems ? [...this._savedItems] : this._items; + this._checked = this._savedChecked + ? new Set(this._savedChecked) + : this._checked; + this._dialog.close(); + } + _selectAll() { this._items.forEach((col) => this._checked.add(col)); this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => { @@ -124,7 +445,7 @@ class ColumnChooser extends HTMLElement { _apply() { const selected = this._items.filter((col) => this._checked.has(col)); - this._modal.close(); + this._dialog.close(); if (this._onApply) { this._onApply(selected); } @@ -151,13 +472,11 @@ class ColumnChooser extends HTMLElement { - + ${col}
      `; - li.querySelector(".drag-item-label").textContent = col; - li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); @@ -190,7 +509,7 @@ class ColumnChooser extends HTMLElement { this._ghostOffX = e.clientX - rect.left; this._ghostOffY = e.clientY - rect.top; - // Keep the drag preview inside the dialog so it stays above the backdrop. + // Build ghost inside shadow DOM this._ghost = document.createElement("div"); this._ghost.className = "drag-ghost"; this._ghost.style.width = rect.width + "px"; @@ -199,7 +518,7 @@ class ColumnChooser extends HTMLElement { this._ghost.querySelector(".drop-indicator")?.remove(); const h = this._ghost.querySelector(".drag-handle"); if (h) h.style.color = "var(--accent)"; - this._modal.dialog.appendChild(this._ghost); + this.shadowRoot.appendChild(this._ghost); srcEl.classList.add("is-dragging"); this._positionGhost(e.clientX, e.clientY); diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 0f61ebd6..9e8b93f6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -915,7 +915,6 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.modal.busy = isSaving; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { @@ -2044,7 +2043,8 @@ async function createTableFromDataPreview(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (tableUrl) { location.href = tableUrl; } else { @@ -2118,7 +2118,8 @@ async function saveTableCreateDialog(state) { var tableUrl = responseData.table_url || fallbackTableUrl(responseData.table || payload.table); - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (tableUrl) { location.href = tableUrl; } else { @@ -2140,6 +2141,18 @@ function confirmDiscardTableCreateChanges(state) { return window.confirm("Discard this new table?"); } +function closeTableCreateDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableCreateChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + function ensureTableCreateDialog(manager) { if (tableCreateDialogState) { return tableCreateDialogState; @@ -2148,8 +2161,7 @@ function ensureTableCreateDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = TABLE_CREATE_DIALOG_ID; dialog.className = "table-create-dialog"; dialog.setAttribute("aria-labelledby", "table-create-title"); @@ -2159,7 +2171,7 @@ function ensureTableCreateDialog(manager) {
      -
      +
      @@ -2186,15 +2198,14 @@ function ensureTableCreateDialog(manager) {
      Create table from data - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); tableCreateDialogState = { - modal: modal, dialog: dialog, form: dialog.querySelector(".table-create-form"), title: dialog.querySelector(".modal-title"), @@ -2214,6 +2225,8 @@ function ensureTableCreateDialog(manager) { manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), + currentButton: null, + shouldRestoreFocus: true, isSaving: false, mode: "manual", dataPreviewRows: null, @@ -2253,7 +2266,7 @@ function ensureTableCreateDialog(manager) { tableCreateDialogState.dataTextarea.focus(); return; } - modal.requestClose("cancel"); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); }); tableCreateDialogState.createFromDataLink.addEventListener( @@ -2351,14 +2364,36 @@ function ensureTableCreateDialog(manager) { updateTableCreateDialogButtons(tableCreateDialogState); }); - modal.beforeClose = function (source) { - return confirmDiscardTableCreateChanges(tableCreateDialogState); - }; + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); dialog.addEventListener("close", function () { var state = tableCreateDialogState; clearTableCreateDialogError(state); setTableCreateDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } }); return tableCreateDialogState; @@ -2379,12 +2414,15 @@ function openTableCreateDialog(button, manager) { menu.open = false; } state.manager = manager; - + state.currentButton = button; + state.shouldRestoreFocus = true; state.title.textContent = "Create a table in " + data.databaseName; clearTableCreateDialogError(state); resetTableCreateDialog(state); loadTableCreateForeignKeyTargets(state); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } state.tableName.focus(); } @@ -2410,7 +2448,6 @@ function initTableCreateActions(manager) { function setRowDeleteDialogBusy(state, isBusy) { state.isBusy = isBusy; - state.modal.busy = isBusy; state.confirmButton.disabled = isBusy; state.cancelButton.disabled = isBusy; state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row"; @@ -2657,7 +2694,6 @@ function showTableAlterDialogError(state, message) { function setTableAlterDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.modal.busy = isSaving; state.cancelButton.disabled = isSaving; state.addColumnButton.disabled = isSaving; state.backButton.disabled = isSaving; @@ -3793,7 +3829,8 @@ async function applyTableAlterChanges(state, result) { result.columnTypeAssignments || [], tableUrl, ); - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (tableAlterResultRenamesTable(result) && tableUrl) { window.location.href = tableUrl; } else { @@ -3854,7 +3891,8 @@ async function dropTableFromAlterDialog(state) { if (!response.ok || (responseData && responseData.ok === false)) { throw rowMutationRequestError(response, responseData); } - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); window.location.href = tableAlterDatabaseUrl() || "/"; } catch (error) { setTableAlterDialogSaving(state, false); @@ -3890,6 +3928,27 @@ function confirmDiscardTableAlterChanges(state) { return window.confirm("Discard table changes?"); } +function closeTableAlterDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableAlterChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function closeTableAlterDialog(state) { + if (!state || state.isSaving) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + function ensureTableAlterDialog(manager) { if (tableAlterDialogState) { return tableAlterDialogState; @@ -3898,8 +3957,7 @@ function ensureTableAlterDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = TABLE_ALTER_DIALOG_ID; dialog.className = "table-alter-dialog"; dialog.setAttribute("aria-labelledby", "table-alter-title"); @@ -3909,7 +3967,7 @@ function ensureTableAlterDialog(manager) {
      -
      +
      - +
      - - - - + + + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); tableAlterDialogState = { - modal: modal, dialog: dialog, form: dialog.querySelector(".table-alter-form"), title: dialog.querySelector(".modal-title"), @@ -3955,6 +4012,8 @@ function ensureTableAlterDialog(manager) { dropButton: dialog.querySelector(".table-alter-drop"), cancelButton: dialog.querySelector(".table-alter-cancel"), saveButton: dialog.querySelector(".table-alter-save"), + currentButton: null, + shouldRestoreFocus: true, isSaving: false, initialSignature: "", originalTableName: "", @@ -3996,7 +4055,7 @@ function ensureTableAlterDialog(manager) { }); tableAlterDialogState.cancelButton.addEventListener("click", function () { - modal.requestClose("cancel"); + closeTableAlterDialog(tableAlterDialogState); }); tableAlterDialogState.dropButton.addEventListener("click", function () { @@ -4017,17 +4076,36 @@ function ensureTableAlterDialog(manager) { } }); - modal.beforeClose = function (source) { - return ( - source === "cancel" || - confirmDiscardTableAlterChanges(tableAlterDialogState) - ); - }; + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); dialog.addEventListener("close", function () { var state = tableAlterDialogState; clearTableAlterDialogError(state); setTableAlterDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } }); return tableAlterDialogState; @@ -4048,7 +4126,8 @@ function openTableAlterDialog(button, manager) { menu.open = false; } state.manager = manager; - + state.currentButton = button; + state.shouldRestoreFocus = true; state.title.textContent = "Alter table " + data.tableName; clearTableAlterDialogError(state); resetTableAlterDialog(state, data); @@ -4058,7 +4137,9 @@ function openTableAlterDialog(button, manager) { tableAlterForeignKeyTargetsUrl(), { filterByType: false }, ); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } var firstName = state.columnList.querySelector(".table-alter-column-name"); if (firstName) { firstName.focus(); @@ -4361,8 +4442,7 @@ function ensureRowDeleteDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = ROW_DELETE_DIALOG_ID; dialog.className = "row-delete-dialog"; dialog.setAttribute("aria-labelledby", "row-delete-title"); @@ -4374,14 +4454,13 @@ function ensureRowDeleteDialog(manager) {

      Delete row ?

      - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); rowDeleteDialogState = { - modal: modal, dialog: dialog, title: dialog.querySelector(".modal-title"), message: dialog.querySelector(".row-delete-message"), @@ -4394,10 +4473,21 @@ function ensureRowDeleteDialog(manager) { currentPkPath: null, manager: manager, isBusy: false, + shouldRestoreFocus: true, }; rowDeleteDialogState.cancelButton.addEventListener("click", function () { - modal.requestClose("cancel"); + if (!rowDeleteDialogState.isBusy) { + rowDeleteDialogState.shouldRestoreFocus = true; + dialog.close(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog && !rowDeleteDialogState.isBusy) { + rowDeleteDialogState.shouldRestoreFocus = true; + dialog.close(); + } }); dialog.addEventListener("keydown", function (ev) { @@ -4409,6 +4499,25 @@ function ensureRowDeleteDialog(manager) { if (!rowDeleteDialogState.isBusy) { rowDeleteDialogState.confirmButton.click(); } + return; + } + if (ev.key !== "Escape") { + return; + } + if (rowDeleteDialogState.isBusy) { + ev.preventDefault(); + return; + } + ev.preventDefault(); + rowDeleteDialogState.shouldRestoreFocus = true; + dialog.close(); + }); + + dialog.addEventListener("cancel", function (ev) { + if (rowDeleteDialogState.isBusy) { + ev.preventDefault(); + } else { + rowDeleteDialogState.shouldRestoreFocus = true; } }); @@ -4416,6 +4525,13 @@ function ensureRowDeleteDialog(manager) { var state = rowDeleteDialogState; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } }); rowDeleteDialogState.confirmButton.addEventListener( @@ -4442,7 +4558,8 @@ function ensureRowDeleteDialog(manager) { throw rowMutationRequestError(response, data); } if (data && data.redirect) { - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); location.href = data.redirect; return; } @@ -4454,7 +4571,8 @@ function ensureRowDeleteDialog(manager) { var statusMessage = state.currentPkPath ? "Deleted row " + state.currentPkPath + "." : "Deleted row."; - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); state.currentRow.remove(); showRowMutationStatus(state.manager, statusMessage, false); if (focusTarget && document.contains(focusTarget)) { @@ -4483,9 +4601,11 @@ function openRowDeleteDialog(button, manager) { } state.manager = manager; + state.currentButton = button; state.currentRow = row; state.currentDeleteUrl = rowDeleteUrl(row); state.currentPkPath = rowDisplayLabel(row); + state.shouldRestoreFocus = true; clearRowDeleteDialogError(state); setRowDeleteDialogBusy(state, false); @@ -4497,7 +4617,9 @@ function openRowDeleteDialog(button, manager) { ); state.rowId.textContent = state.currentPkPath || "this row"; - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } state.confirmButton.focus(); } @@ -5572,7 +5694,6 @@ function setRowEditDialogLoading(state, isLoading) { function setRowEditDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.modal.busy = isSaving; updateRowEditDialogButtons(state); } @@ -5790,6 +5911,18 @@ function confirmDiscardRowEditChanges(state) { return window.confirm(message); } +function closeRowEditDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardRowEditChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + function setRowInsertDialogTitle(state) { var insertData = tableInsertData() || {}; var title = rowEditIsMultipleInsert(state) @@ -6615,6 +6748,38 @@ async function insertBulkPreviewRows(state) { } } +function scheduleCloseRowEditDialogIfConfirmed(state) { + // Fix for an issue in Safari where hitting Esc would show + // the confirm() prompt asking if state should be discarded + // but the Esc key press would then cancel that dialog too. + // Wait for keyup, then move the confirm() to a fresh timer tick. + if (!state || state.isSaving || state.isClosePending) { + return false; + } + if (!rowEditDialogHasChanges(state)) { + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; + } + state.isClosePending = true; + var closeAfterKeyup = function () { + if (!state.isClosePending) { + return; + } + state.isClosePending = false; + closeRowEditDialogIfConfirmed(state); + }; + var onKeyup = function (ev) { + if (ev.key !== "Escape") { + return; + } + document.removeEventListener("keyup", onKeyup, true); + setTimeout(closeAfterKeyup, 0); + }; + document.addEventListener("keyup", onKeyup, true); + return true; +} + function findDataRowElement(root, rowId) { var elements = root.querySelectorAll("[data-row]"); for (var i = 0; i < elements.length; i += 1) { @@ -6704,8 +6869,9 @@ async function saveRowEditDialog(state) { } var formValues = collectRowFormValues(state); if (state.mode === "edit" && !Object.keys(formValues).length) { + state.shouldRestoreFocus = true; hideRowMutationStatus(); - state.modal.close(); + state.dialog.close(); return; } var payload = @@ -6738,8 +6904,9 @@ async function saveRowEditDialog(state) { insertedRowData, insertData.primaryKeys || [], ); + state.shouldRestoreFocus = false; if (!insertedRowId) { - state.modal.close({ restoreFocus: false }); + state.dialog.close(); var missingIdStatus = showRowMutationStatus( state.manager, "Inserted row. Refresh the page to see it.", @@ -6755,7 +6922,7 @@ async function saveRowEditDialog(state) { try { insertedRow = await fetchUpdatedRowElement(state); } catch (_error) { - state.modal.close({ restoreFocus: false }); + state.dialog.close(); var refreshFailedStatus = showRowMutationStatus( state.manager, "Inserted row, but could not refresh the table row. Refresh the page to see it.", @@ -6770,7 +6937,7 @@ async function saveRowEditDialog(state) { rowTitleLabel(insertedRow), ); var addedRow = addInsertedRowToPage(insertedRow); - state.modal.close({ restoreFocus: false }); + state.dialog.close(); showRowMutationStatus(state.manager, insertedStatusMessage, false); if (addedRow) { var insertedFocusTarget = @@ -6779,7 +6946,7 @@ async function saveRowEditDialog(state) { insertedFocusTarget.focus(); } } else { - state.modal.close({ restoreFocus: false }); + state.dialog.close(); var filteredStatus = showRowMutationStatus( state.manager, "Inserted row. It does not match the current filters.", @@ -6791,7 +6958,8 @@ async function saveRowEditDialog(state) { } if (isRowPage()) { - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); location.reload(); return; } @@ -6827,7 +6995,8 @@ async function saveRowEditDialog(state) { ); } - state.modal.close({ restoreFocus: false }); + state.shouldRestoreFocus = false; + state.dialog.close(); if (focusTarget && document.contains(focusTarget)) { focusTarget.focus(); } @@ -6971,8 +7140,7 @@ function ensureRowEditDialog(manager) { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = ROW_EDIT_DIALOG_ID; dialog.className = "row-edit-dialog"; dialog.setAttribute("aria-labelledby", "row-edit-title"); @@ -6984,8 +7152,8 @@ function ensureRowEditDialog(manager) {

      Loading row...

      -
      - @@ -7015,15 +7183,14 @@ function ensureRowEditDialog(manager) {
      - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); rowEditDialogState = { - modal: modal, dialog: dialog, form: dialog.querySelector(".row-edit-form"), title: dialog.querySelector(".modal-title"), @@ -7054,6 +7221,7 @@ function ensureRowEditDialog(manager) { singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), + currentButton: null, currentRow: null, currentRowId: null, currentPkPath: null, @@ -7081,7 +7249,9 @@ function ensureRowEditDialog(manager) { manager: manager, isLoading: false, isSaving: false, + isClosePending: false, hasLoaded: false, + shouldRestoreFocus: true, }; rowEditDialogState.form.addEventListener("submit", function (ev) { @@ -7101,7 +7271,10 @@ function ensureRowEditDialog(manager) { rowEditDialogState.bulkInsertTextarea.focus(); return; } - modal.requestClose("cancel"); + if (!rowEditDialogState.isSaving) { + rowEditDialogState.shouldRestoreFocus = true; + dialog.close(); + } }); rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { @@ -7220,17 +7393,31 @@ function ensureRowEditDialog(manager) { }, ); - modal.beforeClose = function (source) { - return ( - source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState) - ); - }; + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeRowEditDialogIfConfirmed(rowEditDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState); + }); dialog.addEventListener("close", function () { var state = rowEditDialogState; var shouldReloadOnClose = state.shouldReloadOnClose; var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; + state.isClosePending = false; state.bulkInsertLiveValidationError = null; state.shouldReloadOnClose = false; state.redirectOnCloseUrl = null; @@ -7243,6 +7430,13 @@ function ensureRowEditDialog(manager) { destroyRowEditFields(state); setRowEditDialogLoading(state, false); setRowEditDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } if (shouldReloadOnClose) { if (redirectOnCloseUrl) { location.href = redirectOnCloseUrl; @@ -7267,6 +7461,7 @@ async function openRowEditDialog(button, manager) { state.manager = manager; state.mode = "edit"; + state.currentButton = button; state.currentRow = row; state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); @@ -7283,7 +7478,7 @@ async function openRowEditDialog(button, manager) { } else { state.form.removeAttribute("action"); } - + state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; var loadId = state.loadId; @@ -7302,7 +7497,9 @@ async function openRowEditDialog(button, manager) { state.summary.textContent = ""; syncRowEditInsertModeUi(state); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } state.cancelButton.focus(); try { @@ -7342,6 +7539,7 @@ function openRowInsertDialog(button, manager) { state.manager = manager; state.mode = "insert"; + state.currentButton = button; state.currentRow = null; state.currentRowId = null; state.currentPkPath = null; @@ -7356,7 +7554,7 @@ function openRowInsertDialog(button, manager) { state.shouldReloadOnClose = false; state.redirectOnCloseUrl = null; resetBulkInsertPreview(state); - + state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; @@ -7378,7 +7576,9 @@ function openRowInsertDialog(button, manager) { state.summary.textContent = ""; syncRowEditInsertModeUi(state); - state.modal.show({ returnFocusTo: button }); + if (!state.dialog.open) { + state.dialog.showModal(); + } renderRowInsertFields(state, insertData); } diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..0e6e2c29 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,56 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === "object" && typeof module !== "undefined" + ? (module.exports = factory()) + : typeof define === "function" && define.amd + ? define(factory) + : (global.jsonFormatHighlight = factory()); +})(this, function () { + "use strict"; + + var defaultColors = { + keyColor: "dimgray", + numberColor: "lightskyblue", + stringColor: "lightcoral", + trueColor: "lightseagreen", + falseColor: "#f66578", + nullColor: "cornflowerblue", + }; + + function index(json, colorOptions) { + if (colorOptions === void 0) colorOptions = {}; + + if (!json) { + return; + } + if (typeof json !== "string") { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, "&").replace(//g, ">"); + return json.replace( + /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, + function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) + ? colors.trueColor + : /false/.test(match) + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; + } + return '' + match + ""; + }, + ); + } + + return index; +}); diff --git a/datasette/static/mobile-column-actions.js b/datasette/static/mobile-column-actions.js index e94ee865..a386b1fc 100644 --- a/datasette/static/mobile-column-actions.js +++ b/datasette/static/mobile-column-actions.js @@ -66,8 +66,7 @@ function initMobileColumnActions(manager) { return; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.className = "mobile-column-actions-dialog"; dialog.id = MOBILE_COLUMN_DIALOG_ID; dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID); @@ -76,13 +75,13 @@ function initMobileColumnActions(manager) { Column actions
      -
      +
      Tap a column to reveal actions. - +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); triggerButton.setAttribute("aria-haspopup", "dialog"); triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID); @@ -92,6 +91,7 @@ function initMobileColumnActions(manager) { var listWrap = dialog.querySelector(".mobile-column-list"); var doneButton = dialog.querySelector(".mobile-column-actions-done"); var expandedSectionId = null; + var shouldRestoreFocus = true; function updateExpandedSection() { Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => { @@ -128,7 +128,16 @@ function initMobileColumnActions(manager) { } function closeDialog(options) { - modal.close(options); + options = options || {}; + shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + triggerButton.setAttribute("aria-expanded", "false"); + if (shouldRestoreFocus) { + triggerButton.focus(); + } + } } function renderDialog() { @@ -157,8 +166,7 @@ function initMobileColumnActions(manager) { topActions.className = "mobile-column-top-actions"; var showAllColumns = document.createElement("a"); - showAllColumns.className = - "modal-btn modal-btn-ghost mobile-column-top-action"; + showAllColumns.className = "btn btn-ghost mobile-column-top-action"; showAllColumns.href = manager.columnActions.showAllColumnsUrl(); showAllColumns.textContent = "Show all columns"; @@ -257,7 +265,9 @@ function initMobileColumnActions(manager) { if (!renderDialog()) { return; } - modal.show({ returnFocusTo: triggerButton }); + if (!dialog.open) { + dialog.showModal(); + } triggerButton.setAttribute("aria-expanded", "true"); var focusTarget = dialog.querySelector(".mobile-column-top-action") || @@ -278,8 +288,22 @@ function initMobileColumnActions(manager) { closeDialog(); }); + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeDialog(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeDialog(); + }); + dialog.addEventListener("close", function () { triggerButton.setAttribute("aria-expanded", "false"); + if (shouldRestoreFocus) { + triggerButton.focus(); + } }); window.addEventListener("resize", function () { diff --git a/datasette/static/modal.js b/datasette/static/modal.js deleted file mode 100644 index ca9ab353..00000000 --- a/datasette/static/modal.js +++ /dev/null @@ -1,167 +0,0 @@ -// Shared lifecycle for native modal dialogs. -(() => { - class DatasetteModal extends HTMLElement { - constructor() { - super(); - this.beforeClose = null; - this._busy = false; - this._restoreFocus = true; - this._returnFocusTo = null; - this._escapeCleanup = null; - this._escapeTimer = null; - } - - static create() { - const modal = document.createElement("datasette-modal"); - modal.appendChild(document.createElement("dialog")); - return modal; - } - - get dialog() { - return this.querySelector(":scope > dialog"); - } - - get busy() { - return this._busy; - } - - set busy(value) { - this._busy = !!value; - if (this.dialog) { - this.dialog.setAttribute("aria-busy", String(this._busy)); - } - } - - connectedCallback() { - const dialog = this.dialog; - if (!dialog) return; - dialog.classList.add("datasette-modal"); - this._listeners?.abort(); - this._listeners = new AbortController(); - const options = { signal: this._listeners.signal }; - let backdropPointerDown = false; - const outside = (event) => { - const rect = dialog.getBoundingClientRect(); - return ( - event.target === dialog && - (event.clientX < rect.left || - event.clientX > rect.right || - event.clientY < rect.top || - event.clientY > rect.bottom) - ); - }; - dialog.addEventListener( - "pointerdown", - (event) => { - backdropPointerDown = outside(event); - }, - options, - ); - dialog.addEventListener( - "click", - (event) => { - if (backdropPointerDown && outside(event)) - this.requestClose("backdrop"); - backdropPointerDown = false; - }, - options, - ); - dialog.addEventListener( - "keydown", - (event) => { - if (event.key !== "Escape" || event.defaultPrevented) return; - // A nested native dialog or plugin picker gets first refusal. - if (event.target.closest("dialog") !== dialog) return; - event.preventDefault(); - if (this.busy || this._escapeCleanup || this._escapeTimer !== null) - return; - // Safari can otherwise use this Escape press to cancel confirm() too. - // Only keyboard dismissals wait for keyup; native cancel events needn't. - const onKeyup = (up) => { - if (up.key !== "Escape") return; - this._escapeCleanup(); - this._escapeCleanup = null; - this._escapeTimer = setTimeout(() => { - this._escapeTimer = null; - this.requestClose("escape"); - }, 0); - }; - this.ownerDocument.addEventListener("keyup", onKeyup, true); - this._escapeCleanup = () => - this.ownerDocument.removeEventListener("keyup", onKeyup, true); - }, - options, - ); - dialog.addEventListener( - "cancel", - (event) => { - if (event.target !== dialog) return; - event.preventDefault(); - if (!this._escapeCleanup && this._escapeTimer === null) - this.requestClose("escape"); - }, - options, - ); - dialog.addEventListener( - "close", - (event) => { - if (event.target !== dialog || dialog.open) return; - this._clearPendingClose(); - this.busy = false; - if (this._restoreFocus && this._returnFocusTo?.isConnected) { - // Menu actions may have become hidden while the dialog was open. - const details = this._returnFocusTo.closest("details:not([open])"); - const target = - details?.querySelector("summary") || this._returnFocusTo; - target.focus({ preventScroll: true }); - } - this._returnFocusTo = null; - }, - options, - ); - } - - disconnectedCallback() { - this._listeners?.abort(); - this._clearPendingClose(); - this._returnFocusTo = null; - if (this.dialog?.open) this.dialog.close(); - this.busy = false; - } - - _clearPendingClose() { - this._escapeCleanup?.(); - this._escapeCleanup = null; - clearTimeout(this._escapeTimer); - this._escapeTimer = null; - } - - show({ returnFocusTo, initialFocus } = {}) { - const dialog = this.dialog; - if (!dialog.open) { - this._clearPendingClose(); - this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement; - this._restoreFocus = true; - dialog.showModal(); - } - if (typeof initialFocus === "function") initialFocus(); - else initialFocus?.focus(); - } - - requestClose(source = "cancel") { - if (!this.dialog.open || this.busy) return false; - if (this.beforeClose && this.beforeClose(source) === false) return false; - this.close(); - return true; - } - - close({ restoreFocus = true } = {}) { - this._clearPendingClose(); - this._restoreFocus = restoreFocus; - this.dialog.close(); - } - } - - customElements.define("datasette-modal", DatasetteModal); - window.DatasetteModal = DatasetteModal; -})(); diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index df2516f9..ec2d23d8 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -10,22 +10,277 @@ class NavigationSearch extends HTMLElement { this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; this.statusId = `navigation-search-status-${this.instanceId}`; this.titleId = `navigation-search-title-${this.instanceId}`; + this.attachShadow({ mode: "open" }); this.selectedIndex = -1; this.matches = []; this.renderedMatches = []; this.debounceTimer = null; - } + this.restoreFocusTarget = null; + this.shouldRestoreFocus = true; - connectedCallback() { - if (this._initialized) return; - this._initialized = true; this.render(); this.setupEventListeners(); } render() { - this.innerHTML = ` - + this.shadowRoot.innerHTML = ` + + +
      @@ -47,22 +302,23 @@ class NavigationSearch extends HTMLElement { >
      -
      +
      ↑ ↓ Navigate Enter Select Esc Close
      - + `; } setupEventListeners() { - const dialog = this.querySelector("dialog"); - const input = this.querySelector(".search-input"); - const closeButton = this.querySelector(".close-search"); - const resultsContainer = this.querySelector(".results-container"); + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); + const closeButton = this.shadowRoot.querySelector(".close-search"); + const resultsContainer = + this.shadowRoot.querySelector(".results-container"); // Global keyboard listener for "/" document.addEventListener("keydown", (e) => { @@ -99,6 +355,8 @@ class NavigationSearch extends HTMLElement { } else if (e.key === "Enter") { e.preventDefault(); this.selectCurrentItem(); + } else if (e.key === "Escape") { + this.closeMenu(); } }); @@ -122,6 +380,18 @@ class NavigationSearch extends HTMLElement { } }); + // Close on backdrop click + dialog.addEventListener("click", (e) => { + if (e.target === dialog) { + this.closeMenu(); + } + }); + + dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this.closeMenu(); + }); + dialog.addEventListener("close", () => { this.onMenuClosed(); }); @@ -162,6 +432,19 @@ class NavigationSearch extends HTMLElement { } } + focusRestoreTarget(trigger) { + if (trigger && typeof trigger.focus === "function") { + return trigger; + } + if ( + document.activeElement && + typeof document.activeElement.focus === "function" + ) { + return document.activeElement; + } + return null; + } + setNavigationTriggersExpanded(expanded) { if (typeof document.querySelectorAll !== "function") { return; @@ -182,8 +465,8 @@ class NavigationSearch extends HTMLElement { } updateComboboxState() { - const dialog = this.querySelector("dialog"); - const input = this.querySelector(".search-input"); + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); const matches = this.renderedMatches || []; this.setElementAttribute( input, @@ -208,7 +491,7 @@ class NavigationSearch extends HTMLElement { } setStatus(message) { - const status = this.querySelector(`#${this.statusId}`); + const status = this.shadowRoot.querySelector(`#${this.statusId}`); if (status) { status.textContent = message || ""; } @@ -418,7 +701,7 @@ class NavigationSearch extends HTMLElement { section.render(node, { navigationSearch: this, container, - input: this.querySelector(".search-input"), + input: this.shadowRoot.querySelector(".search-input"), }); }); } @@ -457,8 +740,8 @@ class NavigationSearch extends HTMLElement { } renderResults() { - const container = this.querySelector(".results-container"); - const input = this.querySelector(".search-input"); + const container = this.shadowRoot.querySelector(".results-container"); + const input = this.shadowRoot.querySelector(".search-input"); const showStartContent = !input.value.trim(); const jumpSections = showStartContent ? this.jumpSections() : []; const startBlock = showStartContent @@ -570,15 +853,18 @@ class NavigationSearch extends HTMLElement { } } - openMenu(returnFocusTo) { - const input = this.querySelector(".search-input"); + openMenu(trigger) { + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); - this.querySelector("datasette-modal").show({ - returnFocusTo, - initialFocus: input, - }); + this.restoreFocusTarget = this.focusRestoreTarget(trigger); + this.shouldRestoreFocus = true; + if (!dialog.open) { + dialog.showModal(); + } this.setNavigationTriggersExpanded(true); input.value = ""; + input.focus(); // Reset state, then populate the default jump list. this.matches = []; @@ -588,15 +874,29 @@ class NavigationSearch extends HTMLElement { } closeMenu(options = {}) { - this.querySelector("datasette-modal").close(options); + const dialog = this.shadowRoot.querySelector("dialog"); + this.shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + this.onMenuClosed(); + } } onMenuClosed() { - const input = this.querySelector(".search-input"); + const input = this.shadowRoot.querySelector(".search-input"); this.setElementAttribute(input, "aria-expanded", "false"); this.removeElementAttribute(input, "aria-activedescendant"); this.setNavigationTriggersExpanded(false); this.setStatus(""); + if ( + this.shouldRestoreFocus && + this.restoreFocusTarget && + typeof this.restoreFocusTarget.focus === "function" + ) { + this.restoreFocusTarget.focus(); + } + this.restoreFocusTarget = null; } escapeHtml(text) { diff --git a/datasette/static/table.js b/datasette/static/table.js index 6b3e619c..2bc9fda6 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -157,7 +157,6 @@ function createSetColumnTypeOption(value, name, description, checked) { function setSetColumnTypeDialogBusy(state, isBusy) { state.isBusy = isBusy; - state.modal.busy = isBusy; state.saveButton.disabled = isBusy; state.cancelButton.disabled = isBusy; Array.from( @@ -186,8 +185,7 @@ function ensureSetColumnTypeDialog() { return null; } - var modal = DatasetteModal.create(); - var dialog = modal.dialog; + var dialog = document.createElement("dialog"); dialog.id = SET_COLUMN_TYPE_DIALOG_ID; dialog.className = "set-column-type-dialog"; dialog.setAttribute("aria-labelledby", "set-column-type-title"); @@ -198,17 +196,16 @@ function ensureSetColumnTypeDialog() {

      -
      +
      - - + +
      `; - document.body.appendChild(modal); + document.body.appendChild(dialog); setColumnTypeDialogState = { - modal: modal, dialog: dialog, meta: dialog.querySelector(".modal-meta"), status: dialog.querySelector(".set-column-type-status"), @@ -223,7 +220,21 @@ function ensureSetColumnTypeDialog() { }; setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - modal.requestClose("cancel"); + if (!setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + if (setColumnTypeDialogState.isBusy) { + ev.preventDefault(); + } }); dialog.addEventListener("close", function () { @@ -231,52 +242,49 @@ function ensureSetColumnTypeDialog() { setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); }); - setColumnTypeDialogState.saveButton.addEventListener( - "click", - async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; + setColumnTypeDialogState.saveButton.addEventListener("click", async function () { + var state = setColumnTypeDialogState; + var selected = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + var selectedType = selected ? selected.value : ""; + var currentType = state.currentConfig.current + ? state.currentConfig.current.type + : ""; - if (selectedType === currentType) { - state.modal.close(); - return; + if (selectedType === currentType) { + state.dialog.close(); + return; + } + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, true); + + var payload = { + column: state.currentColumn, + column_type: selectedType ? { type: selectedType } : null, + }; + + try { + var response = await fetch(getSetColumnTypeData().path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + var message = (data.errors || ["Request failed"]).join(" "); + throw new Error(message); } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); - } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }, - ); + location.reload(); + } catch (error) { + setSetColumnTypeDialogBusy(state, false); + showSetColumnTypeDialogError(state, error.message || "Request failed"); + } + }); return setColumnTypeDialogState; } @@ -333,7 +341,9 @@ function openSetColumnTypeDialog(th) { state.optionsWrap.appendChild(emptyState); } - state.modal.show(); + if (!state.dialog.open) { + state.dialog.showModal(); + } var selectedOption = state.dialog.querySelector( 'input[name="set-column-type-choice"]:checked', ); @@ -357,10 +367,9 @@ function shouldShowShowAllColumns() { function hasMultipleVisibleColumns(manager) { return ( - Array.from( - document.querySelectorAll(manager.selectors.tableHeaders), - ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1") - .length > 1 + Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter( + (th) => th.dataset.column && th.dataset.isLinkColumn !== "1", + ).length > 1 ); } @@ -497,6 +506,45 @@ function renderActionLink(itemConfig) { return newLink; } +function initializeStickyTableHeader(tableWrapper) { + var table = tableWrapper.querySelector("table.rows-and-columns"); + var tableHead = table && table.querySelector("thead"); + if (!tableHead || !window.requestAnimationFrame) { + return; + } + + var animationFrame = null; + function updatePosition() { + animationFrame = null; + if (window.matchMedia("(max-width: 576px)").matches) { + tableHead.style.transform = ""; + return; + } + // overflow-x: auto makes tableWrapper the sticky scroll container, even + // though the page handles vertical scrolling. Counter the page scroll so + // the real header stays aligned with both the viewport and the table. + var wrapperTop = tableWrapper.getBoundingClientRect().top; + var maximumOffset = Math.max( + 0, + table.offsetHeight - tableHead.offsetHeight, + ); + var offset = Math.min(Math.max(-wrapperTop, 0), maximumOffset); + tableHead.style.transform = offset + ? `translateY(${offset}px)` + : ""; + } + + function queuePositionUpdate() { + if (animationFrame === null) { + animationFrame = window.requestAnimationFrame(updatePosition); + } + } + + window.addEventListener("scroll", queuePositionUpdate, { passive: true }); + window.addEventListener("resize", queuePositionUpdate); + queuePositionUpdate(); +} + /** Main initialization function for Datasette Table interactions */ const initDatasetteTable = function (manager) { // Feature detection @@ -512,6 +560,7 @@ const initDatasetteTable = function (manager) { if (tableWrapper) { tableWrapper.addEventListener("scroll", closeMenu); } + window.addEventListener("scroll", closeMenu, { passive: true }); document.body.addEventListener("click", (ev) => { /* was this click outside the menu? */ var target = ev.target; @@ -622,6 +671,9 @@ const initDatasetteTable = function (manager) { icon.addEventListener("click", onTableHeaderClick); th.appendChild(icon); }); + if (tableWrapper) { + initializeStickyTableHeader(tableWrapper); + } }; function filterRowSelector(manager) { @@ -640,12 +692,10 @@ function filterRowNumberFromName(name) { } function nextFilterRowNumber(manager) { - return ( - filterRowsWithControls(manager).reduce((max, row) => { - var column = row.querySelector("select"); - return Math.max(max, filterRowNumberFromName(column && column.name)); - }, 0) + 1 - ); + return filterRowsWithControls(manager).reduce((max, row) => { + var column = row.querySelector("select"); + return Math.max(max, filterRowNumberFromName(column && column.name)); + }, 0) + 1; } function setFilterRowNumber(row, number) { @@ -672,11 +722,9 @@ function updateFilterRowButtons(manager) { if (addButton) { addButton.hidden = index !== rows.length - 1 || !column.value; } - var visibleButtonCount = [removeButton, addButton].filter( - function (button) { - return button && !button.hidden; - }, - ).length; + var visibleButtonCount = [removeButton, addButton].filter(function (button) { + return button && !button.hidden; + }).length; row.classList.toggle( "filter-controls-row-has-buttons", visibleButtonCount > 0, @@ -698,9 +746,7 @@ function cloneFilterRow(row) { clone.querySelector(".filter-op select").name = "_filter_op"; clone.querySelector("input.filter-value").name = "_filter_value"; resetFilterRow(clone); - clone - .querySelectorAll(".filter-row-icon") - .forEach((button) => button.remove()); + clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove()); return clone; } @@ -857,45 +903,10 @@ function openColumnChooser() { }); } -function initCountAll() { - var button = document.querySelector(".count-all"); - if (!button) { - return; - } - button.addEventListener("click", async function () { - var count = document.querySelector(".table-count"); - var error = document.querySelector(".count-error"); - button.disabled = true; - button.textContent = "Counting…"; - error.textContent = ""; - try { - var response = await fetch(button.dataset.countUrl + location.search, { - method: "POST", - headers: { - Accept: "application/json", - }, - }); - var data = await response.json(); - if (!response.ok || !data.ok) { - throw new Error((data.errors || ["Count failed"]).join(" ")); - } - count.textContent = - data.count.toLocaleString("en-US") + - (data.count === 1 ? " row" : " rows"); - button.remove(); - } catch (ex) { - error.textContent = ex.message || "Count failed"; - button.disabled = false; - button.textContent = "count all"; - } - }); -} - // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; - initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index db3c6548..f5f977d9 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,9 +1,8 @@ from __future__ import annotations -import json -from collections.abc import Iterable from dataclasses import dataclass -from typing import Any +import json +from typing import Any, Iterable from .utils import tilde_encode, urlsafe_components @@ -387,7 +386,7 @@ async def count_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = f"%{q}%" + params["query_search"] = "%{}%".format(q) if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) @@ -463,7 +462,7 @@ async def list_queries( except ValueError: components = [] if database is None and len(components) == 3: - where_clauses.append(f""" + where_clauses.append(""" ( q.database_name > :cursor_database OR ( @@ -477,12 +476,12 @@ async def list_queries( ) ) ) - """) + """.format(sort_key_sql=sort_key_sql)) params["cursor_database"] = components[0] params["cursor_sort_key"] = components[1] params["cursor_name"] = components[2] elif database is not None and len(components) == 2: - where_clauses.append(f""" + where_clauses.append(""" ( {sort_key_sql} > :cursor_sort_key OR ( @@ -490,7 +489,7 @@ async def list_queries( AND q.name > :cursor_name ) ) - """) + """.format(sort_key_sql=sort_key_sql)) params["cursor_sort_key"] = components[0] params["cursor_name"] = components[1] @@ -503,7 +502,7 @@ async def list_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = f"%{q}%" + params["query_search"] = "%{}%".format(q) if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html index 21a2ea8f..53a824f1 100644 --- a/datasette/templates/_permission_ui_styles.html +++ b/datasette/templates/_permission_ui_styles.html @@ -6,20 +6,8 @@ padding: 1.5em; margin-bottom: 2em; } -.permission-form form { - max-width: 60rem; -} -.permission-form-grid { - display: grid; - gap: 1.5rem; - grid-template-columns: repeat(2, minmax(0, 1fr)); -} -.permission-form-result { - margin-top: 1rem; - max-width: 60rem; -} .form-section { - margin-bottom: 1.25em; + margin-bottom: 1em; } .form-section label { display: block; @@ -27,51 +15,22 @@ font-weight: bold; } .form-section input[type="text"], -.form-section input[type="number"], -.form-section select, -.permission-textarea { - background-color: #fff; - border: 1px solid #aaa; - border-radius: 4px; - box-sizing: border-box; - box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08); - color: #222; - font-family: inherit; - font-size: 1rem; - line-height: 1.4; - max-width: none; - width: 100%; -} -.form-section input[type="text"] { - height: 3rem; - padding: 0.6rem 0.75rem; -} -.form-section input[type="number"] { - height: 3rem; - max-width: 7rem; - padding: 0.6rem 0.75rem; -} .form-section select { - height: 3rem; - padding: 0.6rem 0.75rem; -} -.permission-textarea { - font-family: monospace; - min-height: 12rem; - padding: 0.75rem; - resize: vertical; + width: 100%; + max-width: 500px; + padding: 0.5em; + box-sizing: border-box; + border: 1px solid #ccc; + border-radius: 3px; } .form-section input[type="text"]:focus, -.form-section input[type="number"]:focus, -.form-section select:focus, -.permission-textarea:focus { +.form-section select:focus { + outline: 2px solid #0066cc; border-color: #0066cc; - box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18); - outline: none; } .form-section small { display: block; - margin-top: 0.45em; + margin-top: 0.3em; color: #666; } .form-actions { @@ -183,9 +142,4 @@ text-align: center; color: #666; } -@media only screen and (max-width: 576px) { - .permission-form-grid { - grid-template-columns: minmax(0, 1fr); - } -} diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html index 8e0f486e..d7203c1e 100644 --- a/datasette/templates/_permissions_debug_tabs.html +++ b/datasette/templates/_permissions_debug_tabs.html @@ -44,10 +44,10 @@ diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index fda4032c..1ecc92df 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -3,11 +3,29 @@ {% block title %}Debug allow rules{% endblock %} {% block extra_head %} -{% include "_permission_ui_styles.html" %} {% endblock %} @@ -20,28 +38,24 @@ p.message-warning {

      Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

      -
      -
      -
      -
      - - -
      -
      - - -
      -
      -
      - -
      - +
      +
      +

      + +
      +
      +

      + +
      +
      + +
      + - {% if error %}

      {{ error }}

      {% endif %} +{% if error %}

      {{ error }}

      {% endif %} - {% if result == "True" %}

      Result: allow

      {% endif %} +{% if result == "True" %}

      Result: allow

      {% endif %} - {% if result == "False" %}

      Result: deny

      {% endif %} -
      +{% if result == "False" %}

      Result: deny

      {% endif %} {% endblock %} diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 32686af1..4927cb8d 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,6 +3,7 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} + {% endblock %} {% block content %} @@ -125,7 +126,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -173,7 +174,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index e5aa46f3..18288439 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -8,7 +8,6 @@ {% endfor %} - {% for url in extra_js_urls %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index c73cdfb7..83cc1ae6 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,6 +3,7 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -48,7 +49,7 @@
      - + Number of results per page (max 200)
      @@ -197,7 +198,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } function displayError(data) { @@ -207,7 +208,7 @@ function displayError(data) { resultsContent.innerHTML = `
      Error: ${escapeHtml(data.error || 'Unknown error')}
      `; - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index c0081c66..3b229a25 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,8 +1,9 @@ {% extends "base.html" %} -{% block title %}Explain a permission decision{% endblock %} +{% block title %}Permission Check{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} {% block content %} -

      Explain a permission decision

      +

      Permission check

      {% set current_tab = "check" %} {% include "_permissions_debug_tabs.html" %} -

      Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

      +

      Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

      + +{% if request.actor %} +

      Current actor: {{ request.actor.get("id", "anonymous") }}

      +{% else %} +

      Current actor: anonymous (not logged in)

      +{% endif %}
      -
      +
      - - - Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. -
      - -
      - + - The operation to evaluate + The permission action to check
      -
      - +
      + - The database or other parent resource + For database-level permissions, specify the database name
      -
      - - - The table, query or other child resource +
      + + + For table-level permissions, specify the table name (requires parent)
      - +
      + {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 8b0cbbcf..4410a677 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Permission activity{% endblock %} +{% block title %}Debug permissions{% endblock %} {% block extra_head %} {% include "_permission_ui_styles.html" %} @@ -20,45 +20,60 @@ .check-action, .check-when, .check-result { font-size: 1.3em; } +textarea { + height: 10em; + width: 95%; + box-sizing: border-box; + padding: 0.5em; + border: 2px dotted black; +} +.two-col { + display: inline-block; + width: 48%; +} +.two-col label { + width: 48%; +} +@media only screen and (max-width: 576px) { + .two-col { + width: 100%; + } +} {% endblock %} {% block content %} -

      Permission activity

      +

      Permission playground

      {% set current_tab = "permissions" %} {% include "_permissions_debug_tabs.html" %} -

      Raw simulator

      - -

      This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

      +

      This tool lets you simulate an actor and a permission check for that actor.

      -
      -
      -
      - - -
      +
      +
      + +
      -
      -
      - - -
      -
      - - -
      -
      - - -
      +
      +
      +
      + + +
      +
      + + +
      +
      + +
      @@ -110,7 +125,7 @@ debugPost.addEventListener('submit', function(ev) { }); -

      Recent permission checks

      +

      Recent permissions checks

      {% if filter != "all" %}All{% else %}All{% endif %}, diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html index a74c18f7..d00ba9cc 100644 --- a/datasette/templates/debug_rules.html +++ b/datasette/templates/debug_rules.html @@ -3,6 +3,7 @@ {% block title %}Permission Rules{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -36,7 +37,7 @@

      - + Number of results per page (max 200)
      @@ -184,7 +185,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } function displayError(data) { @@ -194,7 +195,7 @@ function displayError(data) { resultsContent.innerHTML = `
      Error: ${escapeHtml(data.error || 'Unknown error')}
      `; - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 3ce88e35..c2131360 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -47,12 +47,11 @@ {% endif %} {% if count or human_description_en %} -

      - {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows - - +

      + {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows + {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} - {% if human_description_en %}{{ human_description_en }}{% endif %} + {% if human_description_en %}{{ human_description_en }}{% endif %}

      {% endif %} diff --git a/datasette/tokens.py b/datasette/tokens.py index 79f840d2..4f905339 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -10,7 +10,7 @@ from __future__ import annotations import dataclasses import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import itsdangerous @@ -50,24 +50,24 @@ class TokenRestrictions: database: dict[str, list[str]] = dataclasses.field(default_factory=dict) resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict) - def allow_all(self, action: str) -> TokenRestrictions: + def allow_all(self, action: str) -> "TokenRestrictions": """Allow an action across all databases and resources.""" self.all.append(action) return self - def allow_database(self, database: str, action: str) -> TokenRestrictions: + def allow_database(self, database: str, action: str) -> "TokenRestrictions": """Allow an action on a specific database.""" self.database.setdefault(database, []).append(action) return self def allow_resource( self, database: str, resource: str, action: str - ) -> TokenRestrictions: + ) -> "TokenRestrictions": """Allow an action on a specific resource within a database.""" self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self - def abbreviated(self, datasette: Datasette) -> dict | None: + def abbreviated(self, datasette: "Datasette") -> Optional[dict]: """ Return the abbreviated ``_r`` dictionary shape for this set of restrictions, using action abbreviations registered with ``datasette``. @@ -112,16 +112,16 @@ class TokenHandler: async def create_token( self, - datasette: Datasette, + datasette: "Datasette", actor_id: str, *, - expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + expires_after: Optional[int] = None, + restrictions: Optional[TokenRestrictions] = None, ) -> str: """Create and return a token string for the given actor.""" raise NotImplementedError - async def verify_token(self, datasette: Datasette, token: str) -> dict | None: + async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: """ Verify a token and return an actor dict. @@ -142,11 +142,11 @@ class SignedTokenHandler(TokenHandler): async def create_token( self, - datasette: Datasette, + datasette: "Datasette", actor_id: str, *, - expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + expires_after: Optional[int] = None, + restrictions: Optional[TokenRestrictions] = None, ) -> str: if not datasette.setting("allow_signed_tokens"): raise ValueError( @@ -163,7 +163,7 @@ class SignedTokenHandler(TokenHandler): token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) - async def verify_token(self, datasette: Datasette, token: str) -> dict | None: + async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: prefix = "dstok_" if not token.startswith(prefix): @@ -200,8 +200,9 @@ class SignedTokenHandler(TokenHandler): ): duration = max_signed_tokens_ttl - if duration and time.time() - created > duration: - raise TokenInvalid("Token has expired") + if duration: + if time.time() - created > duration: + raise TokenInvalid("Token has expired") actor = {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/tracer.py b/datasette/tracer.py index 1fbda6f9..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,11 +1,10 @@ import asyncio -import json -import time -import traceback from contextlib import contextmanager from contextvars import ContextVar - from markupsafe import escape +import time +import json +import traceback tracers = {} @@ -133,17 +132,17 @@ class AsgiTracer: "num_traces": len(traces), "traces": traces, } - content_type = next( - ( + try: + content_type = [ v.decode("utf8") for k, v in response_headers if k.lower() == b"content-type" - ), - "", - ) + ][0] + except IndexError: + content_type = "" if "text/html" in content_type and b"" in accumulated_body: extra = escape(json.dumps(trace_info, indent=2)) - extra_html = f"
      {extra}
      ".encode() + extra_html = f"
      {extra}
      ".encode("utf8") accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): data = json.loads(accumulated_body.decode("utf8")) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index f8da20f3..16b3d42b 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,7 +1,6 @@ +from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib -from .utils import PrefixedUrlString, path_with_format, tilde_encode - class Urls: def __init__(self, ds): @@ -9,7 +8,8 @@ class Urls: def path(self, path, format=None): if not isinstance(path, PrefixedUrlString): - path = path.removeprefix("/") + if path.startswith("/"): + path = path[1:] path = self.ds.setting("base_url") + path if format is not None: path = path_with_format(path=path, format=format) @@ -56,7 +56,6 @@ class Urls: return PrefixedUrlString(path) def row_blob(self, database, table, row_path, column): - return ( - self.table(database, table) - + f"/{row_path}.blob?_blob_column={urllib.parse.quote_plus(column)}" + return self.table(database, table) + "/{}.blob?_blob_column={}".format( + row_path, urllib.parse.quote_plus(column) ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 669d04c7..42574d3b 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,31 +1,29 @@ import asyncio -import base64 import binascii +from contextlib import contextmanager +import aiofiles +import click +from collections import OrderedDict, namedtuple, Counter import copy import dataclasses +import base64 import hashlib import inspect import json -import os -import re -import secrets -import shlex -import shutil -import tempfile -import time -import types -import typing -import urllib -from collections import Counter, OrderedDict, namedtuple -from collections.abc import Iterable -from contextlib import contextmanager - -import aiofiles -import click import markupsafe import mergedeep +import os +import re +import shlex +import tempfile +import typing +import time +import types +import secrets +import shutil +from typing import Iterable, List, Tuple +import urllib import yaml - from .shutil_backport import copytree from .sqlite import sqlite3, supports_table_xinfo @@ -38,7 +36,7 @@ if typing.TYPE_CHECKING: class PaginatedResources: """Paginated results from allowed_resources query.""" - resources: list["Resource"] + resources: List["Resource"] next: str | None # Keyset token for next page (None if no more results) _datasette: typing.Any = dataclasses.field(default=None, repr=False) _action: str = dataclasses.field(default=None, repr=False) @@ -85,132 +83,22 @@ class PaginatedResources: # From https://www.sqlite.org/lang_keywords.html -reserved_words = { - "abort", - "action", - "add", - "after", - "all", - "alter", - "analyze", - "and", - "as", - "asc", - "attach", - "autoincrement", - "before", - "begin", - "between", - "by", - "cascade", - "case", - "cast", - "check", - "collate", - "column", - "commit", - "conflict", - "constraint", - "create", - "cross", - "current_date", - "current_time", - "current_timestamp", - "database", - "default", - "deferrable", - "deferred", - "delete", - "desc", - "detach", - "distinct", - "drop", - "each", - "else", - "end", - "escape", - "except", - "exclusive", - "exists", - "explain", - "fail", - "for", - "foreign", - "from", - "full", - "glob", - "group", - "having", - "if", - "ignore", - "immediate", - "in", - "index", - "indexed", - "initially", - "inner", - "insert", - "instead", - "intersect", - "into", - "is", - "isnull", - "join", - "key", - "left", - "like", - "limit", - "match", - "natural", - "no", - "not", - "notnull", - "null", - "of", - "offset", - "on", - "or", - "order", - "outer", - "plan", - "pragma", - "primary", - "query", - "raise", - "recursive", - "references", - "regexp", - "reindex", - "release", - "rename", - "replace", - "restrict", - "right", - "rollback", - "row", - "savepoint", - "select", - "set", - "table", - "temp", - "temporary", - "then", - "to", - "transaction", - "trigger", - "union", - "unique", - "update", - "using", - "vacuum", - "values", - "view", - "virtual", - "when", - "where", - "with", - "without", -} +reserved_words = set( + ( + "abort action add after all alter analyze and as asc attach autoincrement " + "before begin between by cascade case cast check collate column commit " + "conflict constraint create cross current_date current_time " + "current_timestamp database default deferrable deferred delete desc detach " + "distinct drop each else end escape except exclusive exists explain fail " + "for foreign from full glob group having if ignore immediate in index " + "indexed initially inner insert instead intersect into is isnull join key " + "left like limit match natural no not notnull null of offset on or order " + "outer plan pragma primary query raise recursive references regexp reindex " + "release rename replace restrict right rollback row savepoint select set " + "table temp temporary then to transaction trigger union unique update using " + "vacuum values view virtual when where with without" + ).split() +) APT_GET_DOCKERFILE_EXTRAS = r""" RUN apt-get update && \ @@ -270,7 +158,7 @@ functions_marked_as_documented = [] def documented(fn=None, *, label=None): def decorate(fn): - fn._datasette_docs_label = label or f"internals_utils_{fn.__name__}" + fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) functions_marked_as_documented.append(fn) return fn @@ -472,7 +360,7 @@ disallawed_sql_res = [ ( re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"), "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( - ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) + ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) ), ) ] @@ -568,7 +456,7 @@ def escape_css_string(s): def escape_sqlite(s): - if _boring_keyword_re.fullmatch(s) and (s.lower() not in reserved_words): + if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): return s return '"{}"'.format(s.replace('"', '""')) @@ -646,7 +534,10 @@ CMD {cmd}""".format( else "" ), environment_variables="\n".join( - [f"ENV {key} '{value}'" for key, value in environment_variables.items()] + [ + "ENV {} '{}'".format(key, value) + for key, value in environment_variables.items() + ] ), install_from=" ".join(install), files=" ".join(files), @@ -745,11 +636,11 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() fks = [] for info in infos: if info is not None: - id, seq, table_name, from_, to_, _on_update, _on_delete, _match = info + id, seq, table_name, from_, to_, on_update, on_delete, match = info fks.append( { "column": from_, @@ -820,8 +711,7 @@ def detect_spatialite(conn): def detect_fts(conn, table): """Detect if table has a corresponding FTS virtual table and return it""" - sql, params = detect_fts_sql(table) - rows = conn.execute(sql, params).fetchall() + rows = conn.execute(detect_fts_sql(table)).fetchall() if len(rows) == 0: return None else: @@ -829,26 +719,18 @@ def detect_fts(conn, table): def detect_fts_sql(table): - escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - return ( - r""" - select name from sqlite_master - where rootpage = 0 - and ( - sql like :fts_double_quoted escape char(92) - or sql like :fts_bracket_quoted escape char(92) - or ( - tbl_name = :table - and sql like '%VIRTUAL TABLE%USING FTS%' - ) + return r""" + select name from sqlite_master + where rootpage = 0 + and ( + sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%' + or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%' + or ( + tbl_name = "{table}" + and sql like '%VIRTUAL TABLE%USING FTS%' ) - """, - { - "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%', - "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%", - "table": table, - }, - ) + ) + """.format(table=table.replace("'", "''")) def detect_json1(conn=None): @@ -859,7 +741,7 @@ def detect_json1(conn=None): try: conn.execute("SELECT json('{}')") return True - except sqlite3.Error: + except Exception: return False finally: if close_conn: @@ -939,7 +821,9 @@ def is_url(value): if not value.startswith("http://") and not value.startswith("https://"): return False # Any whitespace at all is invalid - return not whitespace_re.search(value) + if whitespace_re.search(value): + return False + return True css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$") @@ -992,9 +876,7 @@ def module_from_path(path, name): mod.__file__ = path with open(path, "r") as file: code = compile(file.read(), path, "exec", dont_inherit=True) - # Executing the file is the whole point - this is how --plugins-dir loads - # plugins and how metadata/config .py files are evaluated - exec(code, mod.__dict__) # noqa: S102 + exec(code, mod.__dict__) return mod @@ -1151,7 +1033,9 @@ def escape_fts(query): query += '"' bits = _escape_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) + return " ".join( + '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits + ) class MultiParams: @@ -1163,7 +1047,7 @@ class MultiParams: data[key], (list, tuple) ), "dictionary data should be a dictionary of key => [list]" self._data = data - elif isinstance(data, (list, tuple)): + elif isinstance(data, list) or isinstance(data, tuple): new_data = {} for item in data: assert ( @@ -1253,7 +1137,9 @@ def _gather_arguments(fn, kwargs): for parameter in parameters: if parameter not in kwargs: raise TypeError( - f"{fn} requires parameters {tuple(parameters)}, missing: {set(parameters) - set(kwargs.keys())}" + "{} requires parameters {}, missing: {}".format( + fn, tuple(parameters), set(parameters) - set(kwargs.keys()) + ) ) call_with.append(kwargs[parameter]) return call_with @@ -1322,9 +1208,9 @@ def resolve_env_secrets(config, environ): """Create copy that recursively replaces {"$env": "NAME"} with values from environ""" if isinstance(config, dict): if list(config.keys()) == ["$env"]: - return environ.get(next(iter(config.values()))) + return environ.get(list(config.values())[0]) elif list(config.keys()) == ["$file"]: - with open(next(iter(config.values()))) as fp: + with open(list(config.values())[0]) as fp: return fp.read() else: return { @@ -1420,7 +1306,7 @@ _named_param_re = re.compile(r":(\w+)") @documented -def named_parameters(sql: str) -> list[str]: +def named_parameters(sql: str) -> List[str]: """ Given a SQL statement, return a list of named parameters that are used in the statement @@ -1433,7 +1319,7 @@ def named_parameters(sql: str) -> list[str]: return _named_param_re.findall(sql) -async def derive_named_parameters(db: "Database", sql: str) -> list[str]: +async def derive_named_parameters(db: "Database", sql: str) -> List[str]: """ This undocumented but stable method exists for backwards compatibility with plugins that were using it before it switched to named_parameters() @@ -1457,9 +1343,9 @@ def parse_size_limit(value, default, maximum, name="_size"): if size < 0: raise ValueError except ValueError: - raise ValueError(f"{name} must be a positive integer") + raise ValueError("{} must be a positive integer".format(name)) if size > maximum: - raise ValueError(f"{name} must be <= {maximum}") + raise ValueError("{} must be <= {}".format(name, maximum)) return size @@ -1517,7 +1403,7 @@ class TildeEncoder(dict): elif b == _space: res = "+" else: - res = f"~{b:02X}" + res = "~{:02X}".format(b) self[b] = res return res @@ -1566,13 +1452,7 @@ async def row_sql_params_pks(db, table, pk_values): if use_rowid: select = "rowid, *" pks = ["rowid"] - wheres = [] - for i, pk in enumerate(pks): - escaped_pk = escape_sqlite(pk) - # Preserve the historic always-quoted SQL exposed by _extra=query - if escaped_pk == pk: - escaped_pk = f'"{pk}"' - wheres.append(f"{escaped_pk}=:p{i}") + wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)] sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}" params = {} for i, pk_value in enumerate(pk_values): @@ -1618,7 +1498,7 @@ def _combine(base: dict, update: dict) -> dict: return base -def pairs_to_nested_config(pairs: list[tuple[str, typing.Any]]) -> dict: +def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: """ Parse a list of key-value pairs into a nested dictionary. """ @@ -1633,7 +1513,7 @@ def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm method = getattr(pm.hook, name, None) - assert method is not None, f"No hook found for {name}" + assert method is not None, "No hook found for {}".format(name) async def inner(): html_bits = [] @@ -1657,7 +1537,7 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins_and_allow(source: dict, destination: dict) -> tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: """ Move 'plugins' and 'allow' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys @@ -1744,7 +1624,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict: return { k: ( redact(v) - if not any(pattern in k.casefold() for pattern in key_patterns) + if not any(pattern in k for pattern in key_patterns) else "***" ) for k, v in data.items() diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index 297f5ae5..c7137e6b 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -29,15 +29,6 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks if TYPE_CHECKING: from datasette.app import Datasette - from datasette.permissions import Action - - -def _child_collation(action: "Action") -> str: - """Match resource identity without changing the spelling returned by SQL.""" - resource_class = action.resource_class - if resource_class is not None and resource_class.case_insensitive_child: - return "NOCASE" - return "BINARY" async def build_allowed_resources_sql( @@ -158,7 +149,6 @@ async def _build_single_action_sql( raise ValueError(f"Unknown action: {action}") # Get base resources SQL from the resource class - child_collation = _child_collation(action_obj) base_resources_sql = await action_obj.resource_class.resources_sql( datasette, actor=actor ) @@ -195,7 +185,7 @@ async def _build_single_action_sql( if permission_sql.sql is None: continue rule_sqls.append(f""" - SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( + SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( {permission_sql.sql} ) """.strip()) @@ -262,62 +252,88 @@ async def _build_single_action_sql( ] ) - # Continue with the cascading logic. - # Aggregate the RULES by cascade level (small), rather than grouping - # base x rules (which scales with the number of resources). - def _agg(select_key, where, group_by): - parts = [ - f" SELECT {select_key}", - " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,", - " json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons", - f" FROM all_rules WHERE {where}", - ] - if group_by: - parts.append(f" GROUP BY {group_by}") - return parts - + # Continue with the cascading logic query_parts.extend( - ["child_agg AS ("] - + _agg( - "parent, child,", - "parent IS NOT NULL AND child IS NOT NULL", - "parent, child", - ) - + ["),", "parent_agg AS ("] - + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") - + ["),", "global_agg AS ("] - + _agg("", "parent IS NULL AND child IS NULL", None) - + ["),"] + [ + "child_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", + " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", + " FROM base b", + " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child", + " GROUP BY b.parent, b.child", + "),", + "parent_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", + " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", + " FROM base b", + " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + "global_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", + " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", + " FROM base b", + " LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + ] ) # Add anonymous decision logic if needed if include_is_private: - - def _anon_agg(select_key, where, group_by): - parts = [ - f" SELECT {select_key}", - " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow", - f" FROM anon_rules WHERE {where}", - ] - if group_by: - parts.append(f" GROUP BY {group_by}") - return parts - query_parts.extend( - ["anon_child_agg AS ("] - + _anon_agg( - f"parent, child COLLATE {child_collation} AS child,", - "parent IS NOT NULL AND child IS NOT NULL", - f"parent, child COLLATE {child_collation}", - ) - + ["),", "anon_parent_agg AS ("] - + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") - + ["),", "anon_global_agg AS ("] - + _anon_agg("", "parent IS NULL AND child IS NULL", None) - + ["),"] + [ + "anon_child_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", + " FROM base b", + " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child", + " GROUP BY b.parent, b.child", + "),", + "anon_parent_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", + " FROM base b", + " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + "anon_global_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", + " FROM base b", + " LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + "anon_decisions AS (", + " SELECT", + " b.parent, b.child,", + " CASE", + " WHEN acl.any_deny = 1 THEN 0", + " WHEN acl.any_allow = 1 THEN 1", + " WHEN apl.any_deny = 1 THEN 0", + " WHEN apl.any_allow = 1 THEN 1", + " WHEN agl.any_deny = 1 THEN 0", + " WHEN agl.any_allow = 1 THEN 1", + " ELSE 0", + " END AS anon_is_allowed", + " FROM base b", + " JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))", + " JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))", + " JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))", + "),", + ] ) # Final decisions @@ -326,28 +342,31 @@ async def _build_single_action_sql( "decisions AS (", " SELECT", " b.parent, b.child,", - " -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level", + " -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level", " -- Priority order:", - " -- 1. Child-level deny 2. Child-level allow", - " -- 3. Parent-level deny 4. Parent-level allow", - " -- 5. Global-level deny 6. Global-level allow", + " -- 1. Child-level deny (most specific, blocks access)", + " -- 2. Child-level allow (most specific, grants access)", + " -- 3. Parent-level deny (intermediate, blocks access)", + " -- 4. Parent-level allow (intermediate, grants access)", + " -- 5. Global-level deny (least specific, blocks access)", + " -- 6. Global-level allow (least specific, grants access)", " -- 7. Default deny (no rules match)", " CASE", - " WHEN ca.any_deny = 1 THEN 0", - " WHEN ca.any_allow = 1 THEN 1", - " WHEN pa.any_deny = 1 THEN 0", - " WHEN pa.any_allow = 1 THEN 1", - " WHEN ga.any_deny = 1 THEN 0", - " WHEN ga.any_allow = 1 THEN 1", + " WHEN cl.any_deny = 1 THEN 0", + " WHEN cl.any_allow = 1 THEN 1", + " WHEN pl.any_deny = 1 THEN 0", + " WHEN pl.any_allow = 1 THEN 1", + " WHEN gl.any_deny = 1 THEN 0", + " WHEN gl.any_allow = 1 THEN 1", " ELSE 0", " END AS is_allowed,", " CASE", - " WHEN ca.any_deny = 1 THEN ca.deny_reasons", - " WHEN ca.any_allow = 1 THEN ca.allow_reasons", - " WHEN pa.any_deny = 1 THEN pa.deny_reasons", - " WHEN pa.any_allow = 1 THEN pa.allow_reasons", - " WHEN ga.any_deny = 1 THEN ga.deny_reasons", - " WHEN ga.any_allow = 1 THEN ga.allow_reasons", + " WHEN cl.any_deny = 1 THEN cl.deny_reasons", + " WHEN cl.any_allow = 1 THEN cl.allow_reasons", + " WHEN pl.any_deny = 1 THEN pl.deny_reasons", + " WHEN pl.any_allow = 1 THEN pl.allow_reasons", + " WHEN gl.any_deny = 1 THEN gl.deny_reasons", + " WHEN gl.any_allow = 1 THEN gl.allow_reasons", " ELSE '[]'", " END AS reason", ] @@ -355,34 +374,21 @@ async def _build_single_action_sql( if include_is_private: query_parts.append( - " , CASE WHEN (" - "CASE" - " WHEN aca.any_deny = 1 THEN 0" - " WHEN aca.any_allow = 1 THEN 1" - " WHEN apa.any_deny = 1 THEN 0" - " WHEN apa.any_allow = 1 THEN 1" - " WHEN aga.any_deny = 1 THEN 0" - " WHEN aga.any_allow = 1 THEN 1" - " ELSE 0 END" - ") = 0 THEN 1 ELSE 0 END AS is_private" + " , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private" ) query_parts.extend( [ " FROM base b", - " LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child", - " LEFT JOIN parent_agg pa ON pa.parent = b.parent", - " CROSS JOIN global_agg ga", + " JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))", + " JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))", + " JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))", ] ) if include_is_private: - query_parts.extend( - [ - " LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child", - " LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent", - " CROSS JOIN anon_global_agg aga", - ] + query_parts.append( + " JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))" ) query_parts.append(")") @@ -392,31 +398,10 @@ async def _build_single_action_sql( # Wrap each restriction_sql in a subquery to avoid operator precedence issues # with UNION ALL inside the restriction SQL statements restriction_intersect = "\nINTERSECT\n".join( - f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})" - for sql in restriction_sqls + f"SELECT * FROM ({sql})" for sql in restriction_sqls ) - # Decompose by NULL-pattern so the final filter can use pure-equality - # EXISTS lookups (satisfiable via automatic indexes) instead of a - # correlated OR-scan over the whole list. query_parts.extend( - [ - ",", - "restriction_list AS (", - f" {restriction_intersect}", - "),", - "restriction_exact AS (", - " SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL", - "),", - "restriction_parent_any AS (", - " SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL", - "),", - "restriction_child_any AS (", - " SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL", - "),", - "restriction_all AS (", - " SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1", - ")", - ] + [",", "restriction_list AS (", f" {restriction_intersect}", ")"] ) # Final SELECT @@ -431,11 +416,10 @@ async def _build_single_action_sql( # Add restriction filter if there are restrictions if restriction_sqls: query_parts.append(""" - AND ( - EXISTS (SELECT 1 FROM restriction_all) - OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent) - OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child) - OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child) + AND EXISTS ( + SELECT 1 FROM restriction_list r + WHERE (r.parent = decisions.parent OR r.parent IS NULL) + AND (r.child = decisions.child OR r.child IS NULL) )""") # Add parent filter if specified @@ -491,7 +475,6 @@ async def build_permission_rules_sql( union_parts = [] all_params = {} restriction_sqls = [] - child_collation = _child_collation(action_obj) for permission_sql in permission_sqls: all_params.update(permission_sql.params or {}) @@ -505,7 +488,7 @@ async def build_permission_rules_sql( continue union_parts.append(f""" - SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( + SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM ( {permission_sql.sql} ) """.strip()) @@ -576,7 +559,6 @@ async def check_permissions_for_actions( verdicts = {} for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)): - child_collation = _child_collation(datasette.actions[action]) prefix = f"a{i}_" rule_parts = [] restriction_parts = [] @@ -602,7 +584,7 @@ async def check_permissions_for_actions( if sql is None: continue rule_parts.append( - f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)" + f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)" ) if not rule_parts: @@ -636,8 +618,7 @@ async def check_permissions_for_actions( if restriction_parts: # Database-level restrictions (parent, NULL) match all children restriction_intersect = "\nINTERSECT\n".join( - f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})" - for sql in restriction_parts + f"SELECT * FROM ({sql})" for sql in restriction_parts ) ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)") verdict_sql = f"""({verdict_sql}) AND EXISTS ( @@ -692,240 +673,3 @@ async def check_permission_for_resource( child=child, ) return results[action] - - -async def explain_permission_for_resource( - *, - datasette: "Datasette", - actor: dict | None, - action: str, - parent: str | None, - child: str | None, -) -> dict: - """Explain a permission decision for one action and resource. - - This is intended for Datasette's permission debugging tools. It uses the - same ``permission_resources_sql`` hook results and the same resolution - rules as :func:`check_permissions_for_actions`, but also returns the - matching rules, actor restriction results and ``also_requires`` chain. - - The returned dictionary is part of Datasette's unstable debugging API. - """ - - action_obj = datasette.actions.get(action) - if action_obj is None: - raise ValueError(f"Unknown action: {action}") - - explanation = await _explain_single_action( - datasette=datasette, - actor=actor, - action=action, - parent=parent, - child=child, - ) - - required_actions = [] - if action_obj.also_requires: - required = await explain_permission_for_resource( - datasette=datasette, - actor=actor, - action=action_obj.also_requires, - parent=parent, - child=child, - ) - required_actions.append(required) - - explanation["required_actions"] = required_actions - explanation["allowed"] = bool( - explanation["rule_allowed"] - and explanation["restriction_allowed"] - and all(required["allowed"] for required in required_actions) - ) - explanation["summary"] = _permission_explanation_summary(explanation) - return explanation - - -async def _explain_single_action( - *, - datasette: "Datasette", - actor: dict | None, - action: str, - parent: str | None, - child: str | None, -) -> dict: - """Return matching rules and restrictions for a single action.""" - from datasette.utils.permissions import SKIP_PERMISSION_CHECKS - - permission_sqls = await gather_permission_sql_from_hooks( - datasette=datasette, - actor=actor, - action=action, - ) - - if permission_sqls is SKIP_PERMISSION_CHECKS: - return { - "action": action, - "rule_allowed": True, - "restriction_allowed": True, - "winning_scope": "global", - "matched_rules": [ - { - "scope": "global", - "effect": "allow", - "source": "skip_permission_checks", - "reason": "Permission checks were explicitly skipped", - "decisive": True, - "ignored_because": None, - } - ], - "restrictions": [], - } - - db = datasette.get_internal_database() - matched_rules = [] - restrictions = [] - child_collation = _child_collation(datasette.actions[action]) - - for permission_sql in permission_sqls: - params = dict(permission_sql.params or {}) - parent_param = _unused_parameter_name(params, "_explain_parent") - params[parent_param] = parent - child_param = _unused_parameter_name(params, "_explain_child") - params[child_param] = child - - if permission_sql.sql: - rows = await db.execute( - f""" - SELECT parent, child, allow, reason - FROM ({permission_sql.sql}) AS permission_rules - WHERE (parent IS NULL OR parent = :{parent_param}) - AND (child IS NULL OR child COLLATE {child_collation} = :{child_param}) - """, - params, - ) - for row in rows: - specificity = ( - 2 - if row["child"] is not None - else 1 if row["parent"] is not None else 0 - ) - matched_rules.append( - { - "scope": ("resource", "parent", "global")[2 - specificity], - "effect": "allow" if row["allow"] else "deny", - "source": permission_sql.source, - "reason": row["reason"], - "_specificity": specificity, - } - ) - - if permission_sql.restriction_sql: - restriction_row = ( - await db.execute( - f""" - SELECT EXISTS( - SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules - WHERE (parent IS NULL OR parent = :{parent_param}) - AND (child IS NULL OR child COLLATE {child_collation} = :{child_param}) - ) AS resource_is_in_allowlist - """, - params, - ) - ).first() - restriction_allowed = bool(restriction_row[0]) - restrictions.append( - { - "source": permission_sql.source, - "allowed": restriction_allowed, - "reason": params.get("deny") - or ( - "Resource is included in this restriction allowlist" - if restriction_allowed - else "Resource is not included in this restriction allowlist" - ), - } - ) - - matched_rules.sort( - key=lambda rule: ( - -rule["_specificity"], - 0 if rule["effect"] == "deny" else 1, - rule["source"] or "", - rule["reason"] or "", - ) - ) - - if matched_rules: - winning_specificity = matched_rules[0]["_specificity"] - winning_rules = [ - rule - for rule in matched_rules - if rule["_specificity"] == winning_specificity - ] - rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules) - winning_scope = winning_rules[0]["scope"] - else: - winning_specificity = None - rule_allowed = False - winning_scope = None - - for rule in matched_rules: - specificity = rule.pop("_specificity") - if specificity != winning_specificity: - rule["decisive"] = False - rule["ignored_because"] = "A more specific rule matched" - elif not rule_allowed and rule["effect"] == "allow": - rule["decisive"] = False - rule["ignored_because"] = "A deny rule matched at the same scope" - else: - rule["decisive"] = True - rule["ignored_because"] = None - - return { - "action": action, - "rule_allowed": rule_allowed, - "restriction_allowed": all( - restriction["allowed"] for restriction in restrictions - ), - "winning_scope": winning_scope, - "matched_rules": matched_rules, - "restrictions": restrictions, - } - - -def _unused_parameter_name(params: dict, preferred: str) -> str: - """Return a SQL parameter name that is not already in ``params``.""" - candidate = preferred - suffix = 2 - while candidate in params: - candidate = f"{preferred}_{suffix}" - suffix += 1 - return candidate - - -def _permission_explanation_summary(explanation: dict) -> str: - denied_requirement = next( - ( - required - for required in explanation["required_actions"] - if not required["allowed"] - ), - None, - ) - if denied_requirement: - return ( - f"Denied because {explanation['action']} also requires " - f"{denied_requirement['action']}, which was denied." - ) - if not explanation["matched_rules"]: - return "Denied because no permission rule matched this actor and resource." - if not explanation["rule_allowed"]: - return ( - f"Denied by a {explanation['winning_scope']}-level rule. " - "Deny rules take precedence over allow rules at the same scope." - ) - if not explanation["restriction_allowed"]: - return ( - "Denied because the resource is not included in the actor's restrictions." - ) - return f"Allowed by the matching {explanation['winning_scope']}-level rule." diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 2d4a6cff..610b86f2 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,30 +1,28 @@ -import asyncio import json -import re -from http.cookies import Morsel, SimpleCookie -from mimetypes import guess_type -from pathlib import Path -from urllib.parse import parse_qs, parse_qsl, urlunparse - -import aiofiles -import aiofiles.os - +from typing import Optional from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file from datasette.utils.multipart import ( - DEFAULT_MAX_FIELD_SIZE, - DEFAULT_MAX_FIELDS, + parse_form_data, + MultipartParseError, + FormData, DEFAULT_MAX_FILE_SIZE, + DEFAULT_MAX_REQUEST_SIZE, + DEFAULT_MAX_FIELDS, DEFAULT_MAX_FILES, + DEFAULT_MAX_PARTS, + DEFAULT_MAX_FIELD_SIZE, DEFAULT_MAX_MEMORY_FILE_SIZE, DEFAULT_MAX_PART_HEADER_BYTES, DEFAULT_MAX_PART_HEADER_LINES, - DEFAULT_MAX_PARTS, - DEFAULT_MAX_REQUEST_SIZE, DEFAULT_MIN_FREE_DISK_BYTES, - FormData, - MultipartParseError, - parse_form_data, ) +from mimetypes import guess_type +from urllib.parse import parse_qs, urlunparse, parse_qsl +from pathlib import Path +from http.cookies import SimpleCookie, Morsel +import aiofiles +import aiofiles.os +import re # Workaround for adding samesite support to pre 3.8 python Morsel._reserved["samesite"] = "SameSite" @@ -83,19 +81,6 @@ SAMESITE_VALUES = ("strict", "lax", "none") DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB -class _RequestHeaders(dict): - """Incoming headers with lowercase keys and case-insensitive lookups.""" - - def __getitem__(self, key): - return super().__getitem__(key.lower()) - - def get(self, key, default=None): - return super().get(key.lower(), default) - - def __contains__(self, key): - return super().__contains__(key.lower()) - - class Request: def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES): self.scope = scope @@ -103,7 +88,7 @@ class Request: self.max_post_body_bytes = max_post_body_bytes def __repr__(self): - return f'' + return ''.format(self.method, self.url) @property def method(self): @@ -125,10 +110,10 @@ class Request: @property def headers(self): - return _RequestHeaders( - (k.decode("latin-1").lower(), v.decode("latin-1")) + return { + k.decode("latin-1").lower(): v.decode("latin-1") for k, v in self.scope.get("headers") or [] - ) + } @property def host(self): @@ -182,7 +167,7 @@ class Request: if max_bytes is None: max_bytes = self.max_post_body_bytes too_large = PayloadTooLarge( - f"Request body exceeded maximum size of {max_bytes} bytes" + "Request body exceeded maximum size of {} bytes".format(max_bytes) ) if max_bytes: # Reject early if the client declares an oversized body @@ -221,7 +206,7 @@ class Request: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -314,24 +299,12 @@ class AsgiLifespan: while True: message = await receive() if message["type"] == "lifespan.startup": - try: - for fn in self.on_startup: - await fn() - except Exception as e: # noqa: BLE001 - await send( - {"type": "lifespan.startup.failed", "message": str(e)} - ) - return + for fn in self.on_startup: + await fn() await send({"type": "lifespan.startup.complete"}) elif message["type"] == "lifespan.shutdown": - try: - for fn in self.on_shutdown: - await fn() - except Exception as e: # noqa: BLE001 - await send( - {"type": "lifespan.shutdown.failed", "message": str(e)} - ) - return + for fn in self.on_shutdown: + await fn() await send({"type": "lifespan.shutdown.complete"}) return else: @@ -511,8 +484,6 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): await asgi_send_html(send, "404: File not found", 404) return - # Only the actual static-file handler can bypass dynamic response privacy. - inner_static._datasette_static = True return inner_static @@ -558,9 +529,9 @@ class Response: httponly=False, samesite="lax", ): - assert ( - samesite in SAMESITE_VALUES - ), f"samesite should be one of {SAMESITE_VALUES}" + assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format( + SAMESITE_VALUES + ) cookie = SimpleCookie() cookie[key] = value for prop_name, prop_value in ( @@ -652,23 +623,10 @@ class AsgiRunOnFirstRequest: self.asgi = asgi self.on_startup = on_startup self._started = False - # Guards against concurrent early requests interleaving with startup: - # without this, several requests could all observe `_started is - # False` and proceed before any of them finish running the hooks. - self._lock = asyncio.Lock() async def __call__(self, scope, receive, send): - # Leave "lifespan" scope events alone - this shim only exists as a - # fallback for hosts that never send them. It wraps AsgiLifespan, so - # if it ran on_startup here too, a startup exception would escape - # before AsgiLifespan's own try/except got a chance to turn it into - # a lifespan.startup.failed message. - if scope["type"] != "lifespan" and not self._started: - async with self._lock: - # Re-check: another request may have finished startup while - # we were waiting for the lock. - if not self._started: - for hook in self.on_startup: - await hook() - self._started = True + if not self._started: + self._started = True + for hook in self.on_startup: + await hook() return await self.asgi(scope, receive, send) diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index 0469d7a8..c4b64908 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -13,7 +13,7 @@ Originally shared here: https://www.djangosnippets.org/snippets/1431/ """ -class BaseConverter: +class BaseConverter(object): decimal_digits = "0123456789" def __init__(self, digits): diff --git a/datasette/utils/check_callable.py b/datasette/utils/check_callable.py index e21a769b..a0997d20 100644 --- a/datasette/utils/check_callable.py +++ b/datasette/utils/check_callable.py @@ -1,6 +1,6 @@ import inspect import types -from typing import Any, NamedTuple +from typing import NamedTuple, Any class CallableStatus(NamedTuple): @@ -19,7 +19,7 @@ def check_callable(obj: Any) -> CallableStatus: if isinstance(obj, types.FunctionType): return CallableStatus(True, inspect.iscoroutinefunction(obj)) - if callable(obj): + if hasattr(obj, "__call__"): return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__)) - assert False, f"obj {obj!r} is somehow callable with no __call__ method" + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 0ddeb847..10ca32a5 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -3,7 +3,7 @@ import textwrap from sqlite_utils import Database as SQLiteUtilsDatabase from sqlite_utils import Migrations -from datasette.utils import escape_sqlite, table_column_details +from datasette.utils import table_column_details INTERNAL_DB_SCHEMA_TABLES = { "catalog_databases", @@ -180,9 +180,29 @@ async def init_internal_db(db): await db.execute_write_fn(apply_migrations, transaction=False) -async def populate_schema_tables(internal_db, db, schema_version): +async def populate_schema_tables(internal_db, db): database_name = db.name + def delete_everything(conn): + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_views WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) + + await internal_db.execute_write_fn(delete_everything) + tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows @@ -207,30 +227,25 @@ async def populate_schema_tables(internal_db, db, schema_version): columns = table_column_details(conn, table_name) columns_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **column._asdict(), } for column in columns ) foreign_keys = conn.execute( - f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" + f"PRAGMA foreign_key_list([{table_name}])" ).fetchall() foreign_keys_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **dict(foreign_key), } for foreign_key in foreign_keys ) - indexes = conn.execute( - f"PRAGMA index_list({escape_sqlite(table_name)})" - ).fetchall() + indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() indexes_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **dict(index), } for index in indexes @@ -251,76 +266,47 @@ async def populate_schema_tables(internal_db, db, schema_version): indexes_to_insert, ) = await db.execute_fn(collect_info) - def replace_catalog(conn): - # Delete child rows before their catalog_tables parents so this also - # works if a prepare_connection plugin enables foreign key enforcement. - for table in ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - ): - conn.execute( - f"DELETE FROM {table} WHERE database_name = ?", - [database_name], - ) - conn.execute( - """ - INSERT OR REPLACE INTO catalog_databases ( - database_name, path, is_memory, schema_version - ) VALUES (?, ?, ?, ?) - """, - [ - database_name, - str(db.path) if db.path is not None else None, - db.is_memory, - schema_version, - ], + await internal_db.execute_write_many( + """ + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_views (database_name, view_name, rootpage, sql) + values (?, ?, ?, ?) + """, + views_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden ) - conn.executemany( - """ - INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - tables_to_insert, + """, + columns_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match ) - conn.executemany( - """ - INSERT INTO catalog_views (database_name, view_name, rootpage, sql) - values (?, ?, ?, ?) - """, - views_to_insert, + """, + foreign_keys_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial ) - conn.executemany( - """ - INSERT INTO catalog_columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden - ) - """, - columns_to_insert, - ) - conn.executemany( - """ - INSERT INTO catalog_foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match - ) - """, - foreign_keys_to_insert, - ) - conn.executemany( - """ - INSERT INTO catalog_indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial - ) - """, - indexes_to_insert, - ) - - await internal_db.execute_write_fn(replace_catalog) + """, + indexes_to_insert, + ) diff --git a/datasette/utils/multipart.py b/datasette/utils/multipart.py index 182c7ab1..cfa77486 100644 --- a/datasette/utils/multipart.py +++ b/datasette/utils/multipart.py @@ -11,10 +11,15 @@ Supports: import asyncio import shutil import tempfile -from collections.abc import Callable from dataclasses import dataclass, field from typing import ( Any, + Callable, + Dict, + List, + Optional, + Tuple, + Union, ) from urllib.parse import parse_qsl @@ -24,7 +29,7 @@ DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024 # 100MB DEFAULT_MAX_FIELDS = 1000 DEFAULT_MAX_FILES = 100 # If max_parts is not specified, it defaults to max_fields + max_files -DEFAULT_MAX_PARTS: int | None = None +DEFAULT_MAX_PARTS: Optional[int] = None DEFAULT_MAX_FIELD_SIZE = 100 * 1024 # 100KB DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024 # 1MB DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024 # 16KB @@ -35,6 +40,8 @@ DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024 # 50MB class MultipartParseError(Exception): """Raised when multipart parsing fails.""" + pass + @dataclass class UploadedFile: @@ -50,7 +57,7 @@ class UploadedFile: name: str filename: str - content_type: str | None + content_type: Optional[str] size: int _file: tempfile.SpooledTemporaryFile = field(repr=False) @@ -79,8 +86,7 @@ class UploadedFile: def __del__(self): try: self._file.close() - except Exception: # noqa: BLE001, S110 - # __del__ must never raise + except Exception: pass @@ -92,27 +98,27 @@ class FormData: """ def __init__(self): - self._data: list[tuple[str, str | UploadedFile]] = [] + self._data: List[Tuple[str, Union[str, UploadedFile]]] = [] - def append(self, key: str, value: str | UploadedFile) -> None: + def append(self, key: str, value: Union[str, UploadedFile]) -> None: """Add a key-value pair.""" self._data.append((key, value)) - def __getitem__(self, key: str) -> str | UploadedFile: + def __getitem__(self, key: str) -> Union[str, UploadedFile]: """Get the first value for a key.""" for k, v in self._data: if k == key: return v raise KeyError(key) - def get(self, key: str, default: Any = None) -> str | UploadedFile | None: + def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]: """Get the first value for a key, or default if not found.""" try: return self[key] except KeyError: return default - def getlist(self, key: str) -> list[str | UploadedFile]: + def getlist(self, key: str) -> List[Union[str, UploadedFile]]: """Get all values for a key.""" return [v for k, v in self._data if k == key] @@ -136,15 +142,15 @@ class FormData: """Return unique keys.""" return list(self) - def items(self) -> list[tuple[str, str | UploadedFile]]: + def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]: """Return all key-value pairs.""" return list(self._data) - def values(self) -> list[str | UploadedFile]: + def values(self) -> List[Union[str, UploadedFile]]: """Return all values.""" return [v for _, v in self._data] - def _uploaded_files(self) -> list[UploadedFile]: + def _uploaded_files(self) -> List[UploadedFile]: """Return UploadedFile instances contained in this form.""" return [v for _, v in self._data if isinstance(v, UploadedFile)] @@ -157,7 +163,7 @@ class FormData: for uploaded in self._uploaded_files(): try: uploaded.close_sync() - except Exception: # noqa: BLE001, S110 + except Exception: # Best-effort cleanup; ignore close errors pass @@ -166,7 +172,7 @@ class FormData: for uploaded in self._uploaded_files(): try: await uploaded.close() - except Exception: # noqa: BLE001, S110 + except Exception: # Best-effort cleanup; ignore close errors pass @@ -183,13 +189,13 @@ class FormData: await self.aclose() -def parse_content_disposition(header: str) -> dict[str, str | None]: +def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: """ Parse Content-Disposition header value. Returns dict with 'name', 'filename' keys (filename may be None). """ - result: dict[str, str | None] = {"name": None, "filename": None} + result: Dict[str, Optional[str]] = {"name": None, "filename": None} # Split on semicolons, handling quoted strings parts = [] @@ -232,8 +238,7 @@ def parse_content_disposition(header: str) -> dict[str, str | None]: from urllib.parse import unquote result["filename"] = unquote(encoded, encoding="utf-8") - except Exception: # noqa: BLE001, S110 - # Malformed RFC 5987 filename* - fall back to the plain filename + except Exception: pass continue @@ -245,19 +250,20 @@ def parse_content_disposition(header: str) -> dict[str, str | None]: if key == "name": result["name"] = value - # Only set filename if filename* hasn't already set it - elif key == "filename" and result["filename"] is None: - # Strip path components (security) - # Handle both Unix and Windows paths - value = value.replace("\\", "/") - if "/" in value: - value = value.rsplit("/", 1)[-1] - result["filename"] = value + elif key == "filename": + # Only set if filename* hasn't already set it + if result["filename"] is None: + # Strip path components (security) + # Handle both Unix and Windows paths + value = value.replace("\\", "/") + if "/" in value: + value = value.rsplit("/", 1)[-1] + result["filename"] = value return result -def parse_content_type(header: str) -> tuple[str, dict[str, str]]: +def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]: """ Parse Content-Type header value. @@ -301,7 +307,7 @@ class MultipartParser: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -342,12 +348,12 @@ class MultipartParser: self._tempdir = tempfile.gettempdir() # Current part state - self.current_headers: dict[str, str] = {} - self.current_file: tempfile.SpooledTemporaryFile | None = None + self.current_headers: Dict[str, str] = {} + self.current_file: Optional[tempfile.SpooledTemporaryFile] = None self.current_body = bytearray() - self.current_name: str | None = None - self.current_filename: str | None = None - self.current_content_type: str | None = None + self.current_name: Optional[str] = None + self.current_filename: Optional[str] = None + self.current_content_type: Optional[str] = None def feed(self, chunk: bytes) -> None: """Feed a chunk of data to the parser.""" @@ -358,13 +364,6 @@ class MultipartParser: self.buffer.extend(chunk) self._process() - def close(self) -> None: - """Discard completed uploads and any file still being received.""" - if self.current_file is not None: - self.current_file.close() - self.current_file = None - self.form_data.close() - def _process(self) -> None: """Process buffered data.""" while True: @@ -455,7 +454,7 @@ class MultipartParser: # Parse header try: line_str = line.decode("utf-8", errors="replace") - except UnicodeDecodeError: + except Exception: line_str = line.decode("latin-1") if ":" in line_str: @@ -482,9 +481,7 @@ class MultipartParser: if self.file_count > self.max_files: raise MultipartParseError("Too many files") if self.handle_files: - # Outlives this method - it is filled in across parser callbacks - # and then handed to the UploadedFile the caller consumes - self.current_file = tempfile.SpooledTemporaryFile( # noqa: SIM115 + self.current_file = tempfile.SpooledTemporaryFile( max_size=self.max_memory_file_size ) else: @@ -584,9 +581,6 @@ class MultipartParser: def _finish_part(self) -> None: """Finalize current part and add to form data.""" if self.current_name is None: - if self.current_file is not None: - self.current_file.close() - self.current_file = None return if self.current_filename is not None: @@ -650,7 +644,7 @@ async def parse_form_data( max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -732,50 +726,29 @@ async def parse_form_data( batch_target = 64 * 1024 batch = bytearray() - async def run_parser(fn, *args): - # Cancellation must not close files while a worker is using them. - task = asyncio.create_task(asyncio.to_thread(fn, *args)) - try: - return await asyncio.shield(task) - except asyncio.CancelledError as cancelled: - try: - while not task.done(): - try: - await asyncio.shield(task) - except asyncio.CancelledError: - continue - task.result() - finally: - raise cancelled - async def flush_batch() -> None: if batch: data = bytes(batch) batch.clear() - await run_parser(parser.feed, data) + await asyncio.to_thread(parser.feed, data) - try: - while True: - message = await receive() - message_type = message.get("type") - if message_type == "http.disconnect": - raise MultipartParseError("Client disconnected during request body") - if message_type is not None and message_type != "http.request": - continue - chunk = message.get("body", b"") - if chunk: - batch.extend(chunk) - if len(batch) >= batch_target: - await flush_batch() - if not message.get("more_body", False): - break + while True: + message = await receive() + message_type = message.get("type") + if message_type == "http.disconnect": + raise MultipartParseError("Client disconnected during request body") + if message_type is not None and message_type != "http.request": + continue + chunk = message.get("body", b"") + if chunk: + batch.extend(chunk) + if len(batch) >= batch_target: + await flush_batch() + if not message.get("more_body", False): + break - await flush_batch() - return await run_parser(parser.finalize) - except BaseException: - # No FormData is returned to the caller to take ownership on failure. - await asyncio.to_thread(parser.close) - raise + await flush_batch() + return await asyncio.to_thread(parser.finalize) else: raise MultipartParseError( diff --git a/datasette/utils/permissions.py b/datasette/utils/permissions.py index 5a8ee8e2..fd1e41a1 100644 --- a/datasette/utils/permissions.py +++ b/datasette/utils/permissions.py @@ -2,9 +2,8 @@ from __future__ import annotations import json +from typing import Any, Dict, Iterable, List, Sequence, Tuple import sqlite3 -from collections.abc import Iterable, Sequence -from typing import Any from datasette.permissions import PermissionSQL from datasette.plugins import pm @@ -16,7 +15,7 @@ SKIP_PERMISSION_CHECKS = object() async def gather_permission_sql_from_hooks( *, datasette, actor: dict | None, action: str -) -> list[PermissionSQL] | object: +) -> List[PermissionSQL] | object: """Collect PermissionSQL objects from the permission_resources_sql hook. Ensures that each returned PermissionSQL has a populated ``source``. @@ -35,7 +34,7 @@ async def gather_permission_sql_from_hooks( hookimpls = hook_caller.get_hookimpls() hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action)) - collected: list[PermissionSQL] = [] + collected: List[PermissionSQL] = [] actor_json = json.dumps(actor) if actor is not None else None actor_id = actor.get("id") if isinstance(actor, dict) else None @@ -72,7 +71,7 @@ def _iter_permission_sql_from_result( if isinstance(result, PermissionSQL): return [result] if isinstance(result, (list, tuple)): - collected: list[PermissionSQL] = [] + collected: List[PermissionSQL] = [] for item in result: collected.extend(_iter_permission_sql_from_result(item, action=action)) return collected @@ -91,7 +90,7 @@ def _iter_permission_sql_from_result( def build_rules_union( actor: dict | None, plugins: Sequence[PermissionSQL] -) -> tuple[str, dict[str, Any]]: +) -> Tuple[str, Dict[str, Any]]: """ Compose plugin SQL into a UNION ALL. @@ -103,10 +102,10 @@ def build_rules_union( The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent Plugin parameters should be prefixed with a unique identifier (e.g., source name). """ - parts: list[str] = [] + parts: List[str] = [] actor_json = json.dumps(actor) if actor else None actor_id = actor.get("id") if actor else None - params: dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} + params: Dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} for p in plugins: # No namespacing - just use plugin params as-is @@ -142,10 +141,10 @@ async def resolve_permissions_from_catalog( plugins: Sequence[Any], action: str, candidate_sql: str, - candidate_params: dict[str, Any] | None = None, + candidate_params: Dict[str, Any] | None = None, *, implicit_deny: bool = True, -) -> list[dict[str, Any]]: +) -> List[Dict[str, Any]]: """ Resolve permissions by embedding the provided *candidate_sql* in a CTE. @@ -169,8 +168,8 @@ async def resolve_permissions_from_catalog( - parent, child, allow, reason, source_plugin, depth - resource (rendered "/parent/child" or "/parent" or "/") """ - resolved_plugins: list[PermissionSQL] = [] - restriction_sqls: list[str] = [] + resolved_plugins: List[PermissionSQL] = [] + restriction_sqls: List[str] = [] for plugin in plugins: if callable(plugin) and not isinstance(plugin, PermissionSQL): @@ -399,11 +398,11 @@ async def resolve_permissions_with_candidates( db, actor: dict | None, plugins: Sequence[Any], - candidates: list[tuple[str, str | None]], + candidates: List[Tuple[str, str | None]], action: str, *, implicit_deny: bool = True, -) -> list[dict[str, Any]]: +) -> List[Dict[str, Any]]: """ Resolve permissions without any external candidate table by embedding the candidates as a UNION of parameterized SELECTs in a CTE. @@ -412,8 +411,8 @@ async def resolve_permissions_with_candidates( actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action """ # Build a small CTE for candidates. - cand_rows_sql: list[str] = [] - cand_params: dict[str, Any] = {} + cand_rows_sql: List[str] = [] + cand_params: Dict[str, Any] = {} for i, (parent, child) in enumerate(candidates): pkey = f"cand_p_{i}" ckey = f"cand_c_{i}" diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index d323f5d6..d1fd1bd7 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -6,7 +6,7 @@ https://github.com/python/cpython/blob/v3.8.3/LICENSE """ import os -from shutil import Error, copy, copy2, copystat +from shutil import copy, copy2, copystat, Error def _copytree( diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 6325e890..1be28982 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -1,8 +1,6 @@ -import sys from dataclasses import dataclass from typing import Literal -from datasette.utils import escape_sqlite from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type SQLOperation = Literal[ @@ -197,16 +195,6 @@ def _allow_authorizer_action(*args): return sqlite3.SQLITE_OK -def _disable_authorizer(conn): - # Python 3.11 added support for unregistering an authorizer using None. - # On Python 3.10, None is installed as the callback instead, and the next - # statement fails with "not authorized" when sqlite3 tries to call it. - if sys.version_info >= (3, 11): - conn.set_authorizer(None) - else: - conn.set_authorizer(_allow_authorizer_action) - - def analyze_sql_tables( conn, sql: str, @@ -220,9 +208,7 @@ def analyze_sql_tables( This function is synchronous and connection-based. It temporarily installs a SQLite authorizer, prepares ``EXPLAIN ``, and returns the operation - callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is - additionally executed inside a rolled-back savepoint so its source-table reads - can be discovered by analyzing a query against the temporary view. + callbacks observed while SQLite compiles the statement. """ operations: dict[OperationKey, set[str]] = {} @@ -427,12 +413,12 @@ def analyze_sql_tables( database=None, table=None, sqlite_schema=sqlite_schema, - target=f"{arg1} {arg2}" if arg2 is not None else arg1, + target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, source=source, ) return sqlite3.SQLITE_OK - action_name = _AUTHORIZER_ACTION_NAMES.get(action, f"SQLITE_{action}") + action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) record( "unknown", "unknown", @@ -495,7 +481,7 @@ def analyze_sql_tables( conn, key.table, schema=key.sqlite_schema ) finally: - _disable_authorizer(conn) + conn.set_authorizer(None) has_schema_operation = any( key.target_type in {"table", "index", "view", "trigger", "virtual-table"} @@ -535,7 +521,9 @@ def analyze_sql_tables( and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS ): return True - return bool(key_is_drop_table_delete(key)) + if key_is_drop_table_delete(key): + return True + return False def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( @@ -546,7 +534,7 @@ def analyze_sql_tables( return None return table_kind_cache[(key.sqlite_schema, key.table)] - analysis = SQLAnalysis( + return SQLAnalysis( operations=tuple( Operation( operation=key.operation, @@ -563,58 +551,3 @@ def analyze_sql_tables( for key, columns in operations.items() ) ) - - # SQLite does not resolve the SELECT body of a view when preparing CREATE - # VIEW, so its authorizer does not report reads from the view's source - # tables. Temporarily create the view, analyze a query against it (which - # does resolve the body), then roll the schema change back. Database-level - # callers use an isolated writable connection for this analysis. - create_view_operations = tuple( - operation - for operation in analysis.operations - if operation.operation == "create" and operation.target_type == "view" - ) - if not create_view_operations: - return analysis - - savepoint = "datasette_analyze_create_view" - conn.execute(f"SAVEPOINT {savepoint}") - try: - conn.execute(sql, params if params is not None else {}) - dependency_reads = [] - for view_operation in create_view_operations: - if view_operation.sqlite_schema is None or view_operation.table is None: - raise sqlite3.OperationalError( - "Could not determine the created view name" - ) - quoted_schema = escape_sqlite(view_operation.sqlite_schema) - quoted_view = escape_sqlite(view_operation.table) - qualified_view = f"{quoted_schema}.{quoted_view}" - view_analysis = analyze_sql_tables( - conn, - f"SELECT * FROM {qualified_view}", - database_name=database_name, - schema_to_database=schema_to_database, - ) - dependency_reads.extend( - operation - for operation in view_analysis.operations - if operation.operation == "read" - and not ( - operation.sqlite_schema == view_operation.sqlite_schema - and operation.table == view_operation.table - ) - ) - finally: - conn.execute(f"ROLLBACK TO {savepoint}") - conn.execute(f"RELEASE {savepoint}") - - existing_operations = set(analysis.operations) - return SQLAnalysis( - operations=analysis.operations - + tuple( - operation - for operation in dependency_reads - if operation not in existing_operations - ) - ) diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 2ae1be9b..4743ae4c 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -15,17 +15,8 @@ if hasattr(sqlite3, "enable_callback_tracebacks"): _cached_sqlite_version = None _cached_supports_returning = None SQLiteTableType = Literal["table", "view", "virtual", "shadow"] -_SQLITE_IDENTIFIER_RE = ( - r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)""" -) _VIRTUAL_TABLE_MODULE_RE = re.compile( - r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?" - + _SQLITE_IDENTIFIER_RE - + r"(?:\s*\.\s*" - + _SQLITE_IDENTIFIER_RE - + r")?\s*\bUSING\b\s*(" - + _SQLITE_IDENTIFIER_RE - + r")", + r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", re.IGNORECASE | re.DOTALL, ) _VIRTUAL_TABLE_SHADOW_SUFFIXES = { @@ -92,58 +83,24 @@ def sqlite_table_type( ) -> SQLiteTableType | None: if supports_table_list(): try: - # Use the "PRAGMA table_list" statement form rather than the - # pragma_table_list(...) table-valued function. The - # table-valued function is resolved like an ordinary relation - # name, so an attacker-created table or view literally named - # "pragma_table_list" can shadow it and spoof the reported - # type (e.g. claiming a virtual table is an ordinary table). - # The PRAGMA statement form is a distinct piece of SQL syntax - # that always invokes SQLite's built-in pragma, so it cannot - # be shadowed by a user-created relation. + query = "select type from pragma_table_list where name = ?" + params: tuple[str, ...] = (table,) if schema is not None: - query = f"PRAGMA {_quote_identifier(schema)}.table_list" - else: - query = "PRAGMA table_list" - cursor = conn.execute(query) - columns = [description[0] for description in cursor.description] - for row in cursor.fetchall(): - record = dict(zip(columns, row)) - if record.get("name") != table: - continue - if schema is not None and record.get("schema") != schema: - continue - row_type = record.get("type") - if row_type in {"table", "view", "virtual", "shadow"}: - return row_type + query += " and schema = ?" + params = (table, schema) + row = conn.execute(query, params).fetchone() + if row is not None and row[0] in {"table", "view", "virtual", "shadow"}: + return row[0] except sqlite3.DatabaseError: pass return _sqlite_table_type_from_schema(conn, table, schema=schema) -def check_structured_write_table(conn, table: str, *, allow_missing=False): - """Validate a row-write target on the connection that will perform the write.""" - # SQLite resolves identifiers case-insensitively. The create API must not - # treat a differently cased existing name as a missing table. - row = conn.execute( - "select name from main.sqlite_master where name = ? collate nocase " - "and type in ('table', 'view')", - (table,), - ).fetchone() - if row is None and allow_missing: - return - if row is not None and sqlite_table_type(conn, row[0]) == "table": - return - # Virtual table modules can interpret row writes as administrative operations. - # Their shadow tables are internal storage, not independently writable data. - raise ValueError("Structured writes require an ordinary table") - - def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]: schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" + "select name, sql from {} where type = 'table'".format(schema_table) ).fetchall() except sqlite3.DatabaseError: return [] @@ -161,63 +118,6 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str] return sorted(hidden_tables) + content_fts_tables -def sqlite_derived_table_dependencies( - conn, *, schema: str | None = "main" -) -> dict[str, str]: - """Return implementation table -> logical/content table dependencies. - - ``PRAGMA table_list`` safely identifies virtual and shadow tables, but - does not report which virtual table owns a shadow table or which table is - named by an FTS ``content=`` option. Derive those relationships from - ``sqlite_master`` DDL and the documented shadow-table suffixes. - - Database errors propagate: failed discovery must not be mistaken for an - empty dependency map and cached as permission to skip inheritance. - """ - schema_table = _sqlite_schema_table(schema) - rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" - ).fetchall() - - table_names = {row[0] for row in rows} - # SQLite identifiers fold ASCII letters only. - identifier_case = str.maketrans( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" - ) - canonical_names = {name.translate(identifier_case): name for name in table_names} - dependencies = {} - for virtual_table, sql in rows: - module = _virtual_table_module(sql) - if module is None: - continue - - # SQLite's documented shadow tables are implementation details of - # their logical virtual table. - for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()): - shadow_table = virtual_table + suffix - if shadow_table in table_names: - dependencies[shadow_table] = virtual_table - - # An external-content FTS table can expose values fetched from its - # content table, so it must also depend on that table's permission. - if module in {"fts3", "fts4", "fts5"}: - content_table = _fts_external_content_table(sql) - if content_table: - dependencies[virtual_table] = content_table - - if module in {"fts5vocab", "fts4aux"}: - source = _fts_vocabulary_source(sql, module, schema or "main") - source = ( - canonical_names.get(source.translate(identifier_case)) - if source - else None - ) - # An unresolved source is itself derived, so the one-hop policy denies it. - dependencies[virtual_table] = source or virtual_table - - return dependencies - - def _sqlite_table_type_from_schema( conn, table: str, @@ -227,7 +127,7 @@ def _sqlite_table_type_from_schema( schema_table = _sqlite_schema_table(schema) try: row = conn.execute( - f"select type, sql from {schema_table} where name = ?", + "select type, sql from {} where name = ?".format(schema_table), (table,), ).fetchone() except sqlite3.DatabaseError: @@ -255,7 +155,7 @@ def _is_known_shadow_table( schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" + "select name, sql from {} where type = 'table'".format(schema_table) ).fetchall() except sqlite3.DatabaseError: return False @@ -274,7 +174,7 @@ def _sqlite_schema_table(schema: str | None) -> str: return "sqlite_master" if schema == "temp": return "sqlite_temp_master" - return f"{_quote_identifier(schema)}.sqlite_master" + return "{}.sqlite_master".format(_quote_identifier(schema)) def _quote_identifier(value: str) -> str: @@ -284,151 +184,10 @@ def _quote_identifier(value: str) -> str: def _virtual_table_module(sql: str | None) -> str | None: if not sql: return None - match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql)) - if match is None: - return None - return _unquote_sql_value(match.group(1)).lower() - - -def _fts_external_content_table(sql: str | None) -> str | None: - """Extract the external ``content=`` table from an FTS declaration.""" - if not sql: - return None - sql = _strip_sql_comments(sql) match = _VIRTUAL_TABLE_MODULE_RE.search(sql) if match is None: return None - open_paren = sql.find("(", match.end()) - if open_paren == -1: - return None - close_paren = sql.rfind(")") - if close_paren <= open_paren: - return None - - for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]): - key, separator, value = argument.partition("=") - if not separator or key.strip().lower() != "content": - continue - return _unquote_sql_value(value.strip()) - return None - - -def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None: - """Resolve a vocabulary source within the current SQLite schema. - - Cross-schema sources cannot be represented by the dependency map and - are conservatively left unresolved. - """ - sql = _strip_sql_comments(sql) - match = _VIRTUAL_TABLE_MODULE_RE.search(sql) - if match is None: - return None - start = sql.find("(", match.end()) - end = sql.rfind(")") - if start < 0 or end <= start: - return None - arguments = [ - _unquote_sql_value(arg.strip()) - for arg in _split_sql_arguments(sql[start + 1 : end]) - ] - expected = 2 if module == "fts5vocab" else 1 - if len(arguments) == expected: - return arguments[0] - if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower(): - return arguments[1] - return None - - -def _split_sql_arguments(arguments: str) -> list[str]: - """Split comma-separated SQLite arguments without splitting quoted text.""" - parts = [] - start = 0 - quote = None - closing_quote = None - index = 0 - while index < len(arguments): - char = arguments[index] - if quote is None: - if char in {"'", '"', "`", "["}: - quote = char - closing_quote = "]" if char == "[" else char - elif char == ",": - parts.append(arguments[start:index]) - start = index + 1 - elif char == closing_quote: - # Single/double/backtick quoting escapes the delimiter by - # doubling it. Square-bracket identifiers do not. - if ( - quote != "[" - and index + 1 < len(arguments) - and arguments[index + 1] == closing_quote - ): - index += 1 - else: - quote = None - closing_quote = None - index += 1 - parts.append(arguments[start:]) - return parts - - -def _strip_sql_comments(sql: str) -> str: - """Remove SQLite comments while preserving quoted strings/identifiers.""" - output = [] - quote = None - closing_quote = None - index = 0 - while index < len(sql): - char = sql[index] - next_char = sql[index + 1] if index + 1 < len(sql) else "" - if quote is None: - if char in {"'", '"', "`", "["}: - quote = char - closing_quote = "]" if char == "[" else char - output.append(char) - elif char == "-" and next_char == "-": - index += 2 - while index < len(sql) and sql[index] not in "\r\n": - index += 1 - output.append(" ") - continue - elif char == "/" and next_char == "*": - index += 2 - while index + 1 < len(sql) and sql[index : index + 2] != "*/": - index += 1 - index = min(index + 2, len(sql)) - output.append(" ") - continue - else: - output.append(char) - else: - output.append(char) - if char == closing_quote: - if ( - quote != "[" - and index + 1 < len(sql) - and sql[index + 1] == closing_quote - ): - output.append(sql[index + 1]) - index += 1 - else: - quote = None - closing_quote = None - index += 1 - return "".join(output) - - -def _unquote_sql_value(value: str) -> str: - if len(value) < 2: - return value - pairs = {"'": "'", '"': '"', "`": "`", "[": "]"} - closing = pairs.get(value[0]) - if closing is None or value[-1] != closing: - return value - unquoted = value[1:-1] - if value[0] != "[": - unquoted = unquoted.replace(closing * 2, closing) - return unquoted + return match.group(1).strip("\"'[]`").lower() def _is_fts_content_virtual_table(sql: str | None) -> bool: diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index e0cb74a7..de7e94af 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -1,10 +1,9 @@ -import json -from urllib.parse import urlencode - from asgiref.sync import async_to_sync +from urllib.parse import urlencode +import json # These wrapper classes pre-date the introduction of -# datasette.client and httpx2 to Datasette. They could +# datasette.client and httpx to Datasette. They could # be removed if the Datasette tests are modified to # call datasette.client directly. diff --git a/datasette/version.py b/datasette/version.py index ad6c77c7..387144e9 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a40" +__version__ = "1.0a36" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index bac3b39e..ed7e175f 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,7 +1,7 @@ +from dataclasses import dataclass import dataclasses import types import typing -from dataclasses import dataclass @dataclass(frozen=True) @@ -74,14 +74,16 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - f"{cls.__name__}.{name} is declared with from_extra() but there is no " - "registered extra of that name" + "{}.{} is declared with from_extra() but there is no " + "registered extra of that name".format(cls.__name__, name) ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - f"{cls.__name__}.{name} is declared with from_extra() but the {name} extra is " - f"not available for scope {cls.extras_scope}" + "{}.{} is declared with from_extra() but the {} extra is " + "not available for scope {}".format( + cls.__name__, name, name, cls.extras_scope + ) ) return extra_class.description or "" diff --git a/datasette/views/base.py b/datasette/views/base.py index 7262edb4..66e14a6d 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -2,20 +2,20 @@ import csv import hashlib import sys +from datasette.utils.asgi import Request from datasette.utils import ( + add_cors_headers, EscapeHtmlWriter, InvalidSql, LimitedWriter, - add_cors_headers, path_from_row_pks, path_with_format, sqlite3, ) from datasette.utils.asgi import ( AsgiStream, - BadRequest, - Request, Response, + BadRequest, ) @@ -35,7 +35,7 @@ class DatasetteError(Exception): self.error_dict = error_dict or {} self.status = status self.message_is_html = message_is_html - # Plain text used for JSON and CSV error responses when message is HTML + # Plain text used for JSON error responses when message is HTML self.plain_message = plain_message @@ -129,10 +129,12 @@ class BaseView: template = environment.select_template(templates) template_context = { **context, - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], + **{ + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + }, } headers = {} if self.has_json_alternate: @@ -149,7 +151,9 @@ class BaseView: template_context["alternate_url_json"] = alternate_url_json headers.update( { - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) } ) return Response.html( @@ -180,7 +184,9 @@ async def stream_csv(datasette, fetch_data, request, database): stream = request.args.get("_stream") # Do not calculate facets or counts: extra_parameters = [ - f"{key}=1" for key in ("_nofacet", "_nocount") if not request.args.get(key) + "{}=1".format(key) + for key in ("_nofacet", "_nocount") + if not request.args.get(key) ] if extra_parameters: # Replace request object with a new one with modified scope @@ -210,6 +216,9 @@ async def stream_csv(datasette, fetch_data, request, database): except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: raise @@ -316,9 +325,8 @@ async def stream_csv(datasette, fetch_data, request, database): else: new_row.append(cell) await writer.writerow(new_row) - except Exception as ex: # noqa: BLE001 - # Streaming CSV: report the error into the response body and stop - sys.stderr.write(f"Caught this error: {ex}\n") + except Exception as ex: + sys.stderr.write("Caught this error: {}\n".format(ex)) sys.stderr.flush() await r.write(str(ex)) return diff --git a/datasette/views/database.py b/datasette/views/database.py index da207601..10dc66ae 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,56 +1,49 @@ +from dataclasses import asdict, dataclass, field +from urllib.parse import parse_qsl, urlencode import asyncio import hashlib import itertools import json +import markupsafe import os import textwrap -from dataclasses import asdict, dataclass, field -from urllib.parse import parse_qsl, urlencode - -import markupsafe +from datasette.extras import extra_names_from_request, ExtraScope from datasette.database import QueryInterrupted -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.plugins import pm from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, stored_query_to_dict +from datasette.write_sql import QueryWriteRejected from datasette.utils import ( - InvalidSql, add_cors_headers, await_me_maybe, - call_with_supported_arguments, error_body, + call_with_supported_arguments, + named_parameters as derive_named_parameters, format_bytes, - is_url, make_slot_function, + tilde_decode, + to_css_class, + validate_sql_select, + is_url, path_with_added_args, path_with_format, path_with_removed_args, sqlite3, - tilde_decode, - to_css_class, truncate_url, - validate_sql_select, + InvalidSql, ) -from datasette.utils import ( - named_parameters as derive_named_parameters, -) -from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response -from datasette.write_sql import QueryWriteRejected +from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden +from datasette.plugins import pm -from . import Context from .base import DatasetteError, View, stream_csv -from .query_helpers import ( - _block_framing, - _ensure_stored_query_execution_permissions, - _table_columns, -) -from .table_create_alter import _create_table_ui_context +from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) +from .table_create_alter import _create_table_ui_context +from . import Context @dataclass @@ -107,7 +100,7 @@ class DatabaseView(View): return response if format_ not in ("html", "json"): - raise NotFound(f"Invalid format: {format_}") + raise NotFound("Invalid format: {}".format(format_)) metadata = await datasette.get_database_metadata(database) @@ -171,7 +164,7 @@ class DatabaseView(View): "label": "Create table", "description": "Create a new table in this database.", "attrs": { - "aria-label": f"Create table in {database}", + "aria-label": "Create table in {}".format(database), "data-database-action": "create-table", }, } @@ -278,7 +271,9 @@ class DatabaseView(View): view_name="database", ), headers={ - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) }, ) @@ -561,7 +556,7 @@ async def database_download(request, datasette): if datasette.cors: add_cors_headers(headers) if db.hash: - etag = f'"{db.hash}"' + etag = '"{}"'.format(db.hash) headers["Etag"] = etag # Has user seen this already? if_none_match = request.headers.get("if-none-match") @@ -648,15 +643,8 @@ class QueryView(View): ok = None redirect_url = None try: - execute_write_kwargs = {"request": request} - if stored_query.is_trusted: - analysis = await db.analyze_sql(stored_query.sql, params_for_query) - if any( - operation.operation == "vacuum" for operation in analysis.operations - ): - execute_write_kwargs["transaction"] = False cursor = await db.execute_write( - stored_query.sql, params_for_query, **execute_write_kwargs + stored_query.sql, params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None @@ -669,9 +657,8 @@ class QueryView(View): ).first() if message_result: message = message_result[0] - except Exception as ex: # noqa: BLE001 - # Stored-query on_success_message_sql is user-authored - message = f"Error running on_success_message_sql: {ex}" + except Exception as ex: + message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: if stored_query.on_success_message: @@ -685,8 +672,7 @@ class QueryView(View): redirect_url = stored_query.on_success_redirect ok = True - except Exception as ex: # noqa: BLE001 - # Stored-query execution is user-authored SQL + except Exception as ex: message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR redirect_url = stored_query.on_error_redirect @@ -820,16 +806,16 @@ class QueryView(View): rows = results.rows except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(f""" + textwrap.dedent("""

      SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

      - + - """).strip(), + """.format(markupsafe.escape(ex.sql))).strip(), title="SQL Interrupted", status=400, message_is_html=True, @@ -845,6 +831,8 @@ class QueryView(View): columns = [] except (sqlite3.OperationalError, InvalidSql) as ex: raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) except DatasetteError: raise @@ -861,13 +849,12 @@ class QueryView(View): raise DatasetteError("?sql= is required", status=400) async def fetch_data_for_csv(request, _next=None): - # Reuse the trusted magic parameter values prepared above. - results = await db.execute(sql, params_for_query, truncate=True) + results = await db.execute(sql, params, truncate=True) data = {"rows": results.rows, "columns": results.columns} return data, None, None return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers: + elif format_ in datasette.renderers.keys(): if not sql: raise DatasetteError("?sql= is required", status=400) data = {"ok": True, "rows": rows, "columns": columns} @@ -959,7 +946,9 @@ class QueryView(View): } headers.update( { - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) } ) metadata = await query_metadata() @@ -1040,7 +1029,9 @@ class QueryView(View): + "?" + urlencode( { - "sql": sql, + **{ + "sql": sql, + }, **named_parameter_values, } ) @@ -1142,11 +1133,9 @@ class QueryView(View): headers=headers, ) else: - assert False, f"Invalid format: {format_}" + assert False, "Invalid format: {}".format(format_) if datasette.cors: add_cors_headers(r.headers) - if stored_query_write and format_ == "html": - _block_framing(r) return r @@ -1245,7 +1234,7 @@ async def display_rows(datasette, database, request, rows, columns): '<Binary: {:,} byte{}>'.format( blob_url, ( - f' title="{formatted}"' + ' title="{}"'.format(formatted) if "bytes" not in formatted else "" ), diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index c4f5e3fe..dd35b127 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -1,7 +1,6 @@ import re from urllib.parse import urlencode -from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3 from datasette.utils.asgi import Response @@ -9,8 +8,8 @@ from datasette.utils.asgi import Response from .base import BaseView from .database import display_rows as display_query_rows from .query_helpers import ( - SQL_PARAMETER_FORM_PREFIX, QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -32,7 +31,15 @@ WRITE_TEMPLATE_LABELS = { "delete": "Delete rows", } WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) -CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" +CREATE_TABLE_TEMPLATE_SQL = "\n".join( + ( + "create table new_table (", + " id integer primary key,", + " name text", + " -- created text default (datetime('now'))", + ")", + ) +) def _parameter_names(columns): @@ -42,11 +49,11 @@ def _parameter_names(columns): base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) base = base.strip("_") or "value" if base[0].isdigit(): - base = f"p_{base}" + base = "p_{}".format(base) name = base index = 2 while name in seen: - name = f"{base}_{index}" + name = "{}_{}".format(base, index) index += 1 seen.add(name) names[column] = name @@ -58,7 +65,7 @@ def _quote_identifier(identifier): def _preferred_where_column(table, columns): - lower_table_id = f"{table.lower()}_id" + lower_table_id = "{}_id".format(table.lower()) return ( next((column for column in columns if column.lower() == "id"), None) or next( @@ -83,15 +90,17 @@ def _insert_template_sql(table, columns): auto_pk = _auto_incrementing_primary_key(columns) insert_columns = [column for column in column_names if column != auto_pk] if not insert_columns: - return f"insert into {_quote_identifier(table)}\ndefault values" + return "insert into {}\ndefault values".format(_quote_identifier(table)) names = _parameter_names(insert_columns) return "\n".join( ( - f"insert into {_quote_identifier(table)} (", - ",\n".join(f" {_quote_identifier(column)}" for column in insert_columns), + "insert into {} (".format(_quote_identifier(table)), + ",\n".join( + " {}".format(_quote_identifier(column)) for column in insert_columns + ), ")", "values (", - ",\n".join(f" :{names[column]}" for column in insert_columns), + ",\n".join(" :{}".format(names[column]) for column in insert_columns), ")", ) ) @@ -105,14 +114,18 @@ def _update_template_sql(table, columns): if not set_columns: return "\n".join( ( - f"update {_quote_identifier(table)}", - f"set {_quote_identifier(where_column)} = :new_{names[where_column]}", - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "update {}".format(_quote_identifier(table)), + "set {} = :new_{}".format( + _quote_identifier(where_column), names[where_column] + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) return "\n".join( ( - f"update {_quote_identifier(table)}", + "update {}".format(_quote_identifier(table)), "set " + ",\n".join( "{}{} = :{}".format( @@ -122,7 +135,9 @@ def _update_template_sql(table, columns): ) for index, column in enumerate(set_columns) ), - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) @@ -133,8 +148,10 @@ def _delete_template_sql(table, columns): where_column = _preferred_where_column(table, column_names) return "\n".join( ( - f"delete from {_quote_identifier(table)}", - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "delete from {}".format(_quote_identifier(table)), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) @@ -385,7 +402,7 @@ class ExecuteWriteView(BaseView): try: execute_write_kwargs = {"request": request} cursor = await db.execute_write(sql, params, **execute_write_kwargs) - except (QueryInterrupted, sqlite3.DatabaseError) as ex: + except sqlite3.DatabaseError as ex: message = str(ex) if wants_json: return _block_framing(Response.error([message], 400)) diff --git a/datasette/views/index.py b/datasette/views/index.py index f73ee38a..67296cd1 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -2,11 +2,11 @@ import json from datasette.plugins import pm from datasette.utils import ( - UNSTABLE_API_MESSAGE, - CustomJSONEncoder, add_cors_headers, await_me_maybe, make_slot_function, + CustomJSONEncoder, + UNSTABLE_API_MESSAGE, ) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -46,15 +46,15 @@ class IndexView(BaseView): databases = [] # Iterate over allowed databases instead of all databases - for name, allowed_db in allowed_db_dict.items(): + for name in allowed_db_dict.keys(): db = self.ds.databases[name] - database_private = allowed_db.private + database_private = allowed_db_dict[name].private # Get allowed tables/views for this database allowed_for_db = tables_by_db.get(name, {}) # Get table names from allowed set instead of db.table_names() - table_names = [child_name for child_name in allowed_for_db] + table_names = [child_name for child_name in allowed_for_db.keys()] hidden_table_names = set(await db.hidden_table_names()) @@ -99,7 +99,7 @@ class IndexView(BaseView): # We will be sorting by number of relationships, so populate that field all_foreign_keys = await db.get_all_foreign_keys() for table, foreign_keys in all_foreign_keys.items(): - if table in tables: + if table in tables.keys(): count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) tables[table]["num_relationships_for_sorting"] = count @@ -121,7 +121,8 @@ class IndexView(BaseView): # Only add views if this is less than TRUNCATE_AT if len(tables_and_views_truncated) < TRUNCATE_AT: num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated) - tables_and_views_truncated.extend(views[:num_views_to_add]) + for view in views[:num_views_to_add]: + tables_and_views_truncated.append(view) databases.append( { diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 725d9cdb..588891d4 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,19 +5,6 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, ) -from datasette.utils import ( - InvalidSql, - escape_sqlite, - parse_size_limit, - path_from_row_pks, - sqlite3, - validate_sql_select, -) -from datasette.utils import ( - named_parameters as derive_named_parameters, -) -from datasette.utils.asgi import Forbidden -from datasette.utils.sql_analysis import Operation, SQLAnalysis from datasette.write_sql import ( IgnoreWriteSqlOperation, QueryWriteRejected, @@ -25,6 +12,17 @@ from datasette.write_sql import ( decision_for_write_sql_operation, operation_is_write, ) +from datasette.utils import ( + parse_size_limit, + named_parameters as derive_named_parameters, + escape_sqlite, + path_from_row_pks, + sqlite3, + validate_sql_select, + InvalidSql, +) +from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -93,7 +91,7 @@ def _as_optional_bool(value, name): return True if lowered in {"0", "false", "f", "no", "off"}: return False - raise QueryValidationError(f"{name} must be 0 or 1") + raise QueryValidationError("{} must be 0 or 1".format(name)) def _query_list_limit(value, default, maximum): @@ -173,7 +171,7 @@ async def _json_or_form_payload(request): try: return json.loads(body or b"{}"), True except json.JSONDecodeError as e: - raise QueryValidationError(f"Invalid JSON: {e}") + raise QueryValidationError("Invalid JSON: {}".format(e)) return await request.post_vars(), False @@ -194,7 +192,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError(f"Could not analyze query: {ex}") from ex + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex is_write = _analysis_is_write(analysis) if is_write: @@ -295,7 +293,8 @@ def _coerce_execute_write_payload(data, is_json): for key, value in data.items(): if key in {"sql", "csrftoken", "_json"}: continue - key = key.removeprefix(SQL_PARAMETER_FORM_PREFIX) + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") @@ -315,7 +314,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError(f"Could not analyze query: {ex}") from ex + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex if not _analysis_is_write(analysis): raise QueryValidationError( "Use /-/query for read-only SQL; this endpoint only executes writes" @@ -497,7 +496,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): ) try: result = await db.execute( - f"select {select} from {escape_sqlite(table)} where rowid = ?", + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), [lastrowid], ) except sqlite3.DatabaseError: diff --git a/datasette/views/row.py b/datasette/views/row.py index 92e75199..c90a3bbe 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,37 +8,34 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils +from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response from datasette.database import QueryInterrupted -from datasette.events import DeleteRowEvent, UpdateRowEvent -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.plugins import pm +from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource +from .base import BaseView, DatasetteError, stream_csv from datasette.utils import ( - CustomJSONEncoder, - CustomRow, - InvalidSql, - WriteJsonValueError, add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, + CustomRow, decode_write_json_row, - escape_sqlite, + InvalidSql, make_slot_function, path_from_row_pks, path_with_format, path_with_removed_args, - sqlite3, - tilde_decode, to_css_class, + escape_sqlite, + sqlite3, + WriteJsonValueError, ) -from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response -from datasette.utils.sqlite import check_structured_write_table - +from datasette.plugins import pm +from datasette.extras import extra_names_from_request, ExtraScope from . import Context, from_extra -from .base import BaseView, DatasetteError, stream_csv from .table import ( - _table_page_data, display_columns_and_rows, + _table_page_data, row_label_from_label_column, ) from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -139,12 +136,6 @@ class RowContext(Context): ) -async def _database_and_table_resource_from_request(datasette, request): - db = await datasette.resolve_database(request) - table = tilde_decode(request.url_vars["table"]) - return db, table, TableResource(database=db.name, table=table) - - class RowView(BaseView): name = "row" @@ -196,16 +187,16 @@ class RowView(BaseView): data, extra_template_data, templates = response_or_template_contexts except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(f""" + textwrap.dedent("""

      SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

      - + - """).strip(), + """.format(markupsafe.escape(ex.sql))).strip(), title="SQL Interrupted", status=400, message_is_html=True, @@ -216,13 +207,15 @@ class RowView(BaseView): ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) except DatasetteError: raise end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - if format_ in self.ds.renderers: + if format_ in self.ds.renderers.keys(): # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -265,13 +258,13 @@ class RowView(BaseView): if status_code is not None: response.status = status_code else: - raise NotFound(f"Invalid format: {format_}") + raise NotFound("Invalid format: {}".format(format_)) ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): ttl = self.ds.setting("default_cache_ttl") - return self.set_response_headers(response, ttl, request) + return self.set_response_headers(response, ttl) async def html(self, request, data, extra_template_data, templates): extras = {} @@ -380,54 +373,42 @@ class RowView(BaseView): view_name=self.name, ), headers={ - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) }, ) - def set_response_headers(self, response, ttl, request=None): - private = getattr(request, "_datasette_private_response", False) + def set_response_headers(self, response, ttl): # Set far-future cache expiry if self.ds.cache_headers and response.status == 200: - if private: - # This response is only visible to the current actor (denied - # to anonymous requests), so it must never be stored by a - # shared cache/CDN - and ?_ttl= must not override that. - response.headers["Cache-Control"] = "private, no-store" - response.headers["Vary"] = "Cookie" + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" else: - ttl = int(ttl) - if ttl == 0: - ttl_header = "no-cache" - else: - ttl_header = f"max-age={ttl}" - response.headers["Cache-Control"] = ttl_header + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header response.headers["Referrer-Policy"] = "no-referrer" if self.ds.cors: add_cors_headers(response.headers) return response async def data(self, request, default_labels=False): - db, table, resource = await _database_and_table_resource_from_request( - self.ds, request - ) + resolved = await self.ds.resolve_row(request) + db = resolved.db database = db.name + table = resolved.table + pk_values = resolved.pk_values - # Check the URL resource before resolving the row, so a denied request - # cannot distinguish an existing primary key from a missing one. + # Ensure user has permission to view this row visible, private = await self.ds.check_visibility( request.actor, action="view-table", - resource=resource, + resource=TableResource(database=database, table=table), ) if not visible: raise Forbidden("You do not have permission to view this table") - # Record whether this response is private (visible to this actor - # only) so set_response_headers() can set appropriate Cache-Control - # headers, regardless of which output format ends up being rendered. - request._datasette_private_response = private - resolved = await self.ds.resolve_row(request) - pk_values = resolved.pk_values results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) @@ -504,8 +485,8 @@ class RowView(BaseView): for row in display_rows: for cell in row: if cell["column"] in pk_set: - cell["value"] = markupsafe.Markup("{}").format( - cell["value"] + cell["value"] = markupsafe.Markup( + "{}".format(cell["value"]) ) label_column = await db.label_column_for_table(table) if is_table else None @@ -519,7 +500,7 @@ class RowView(BaseView): row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = f"{pk_path} {row_label}" + row_action_label = "{} {}".format(pk_path, row_label) row_action_permissions = {} if is_table and db.is_mutable: @@ -532,7 +513,7 @@ class RowView(BaseView): row_actions = [] if row_action_permissions.get("update-row"): attrs = { - "aria-label": f"Edit row {row_action_label}", + "aria-label": "Edit row {}".format(row_action_label), "data-row": row_path, "data-row-action": "edit", } @@ -548,7 +529,7 @@ class RowView(BaseView): ) if row_action_permissions.get("delete-row"): attrs = { - "aria-label": f"Delete row {row_action_label}", + "aria-label": "Delete row {}".format(row_action_label), "data-row": row_path, "data-row-action": "delete", } @@ -578,7 +559,7 @@ class RowView(BaseView): "private": private, "columns": reordered_columns, "foreign_key_tables": await self.foreign_key_tables( - database, table, pk_values, actor=request.actor + database, table, pk_values ), "database_color": db.color, "display_columns": display_columns, @@ -655,23 +636,12 @@ class RowView(BaseView): ), ) - async def foreign_key_tables(self, database, table, pk_values, *, actor): + async def foreign_key_tables(self, database, table, pk_values): if len(pk_values) != 1: return [] db = self.ds.databases[database] all_foreign_keys = await db.get_all_foreign_keys() - foreign_keys = [] - table_permissions = {} - for fk in all_foreign_keys[table]["incoming"]: - other_table = fk["other_table"] - if other_table not in table_permissions: - table_permissions[other_table] = await self.ds.allowed( - action="view-table", - resource=TableResource(database=database, table=other_table), - actor=actor, - ) - if table_permissions[other_table]: - foreign_keys.append(fk) + foreign_keys = all_foreign_keys[table]["incoming"] if len(foreign_keys) == 0: return [] @@ -709,7 +679,7 @@ class RowView(BaseView): key, ",".join(pk_values), ) - foreign_key_tables.append({**fk, "count": count, "link": link}) + foreign_key_tables.append({**fk, **{"count": count, "link": link}}) return foreign_key_tables @@ -728,57 +698,38 @@ def _truncated_row_flash_label(label): return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026" -async def _row_flash_message( - datasette, request, action, resolved, row=None, *, refresh_row=False -): +async def _row_flash_message(db, action, resolved, row=None): pk_label = ", ".join(resolved.pk_values) - # Mutation permission does not grant access to stored row labels. - if not await datasette.allowed( - action="view-table", - resource=TableResource(database=resolved.db.name, table=resolved.table), - actor=request.actor, - ): - return f"{action} row {pk_label}" - - if refresh_row and row is None: - results = await resolved.db.execute( - resolved.sql, resolved.params, truncate=True - ) - row = results.first() - label_column = await resolved.db.label_column_for_table(resolved.table) + label_column = await db.label_column_for_table(resolved.table) label = row_label_from_label_column(row or resolved.row, label_column) if label: label = _truncated_row_flash_label(label) if label and label != pk_label: - return f"{action} row {pk_label} ({label})" - return f"{action} row {pk_label}" + return "{} row {} ({})".format(action, pk_label, label) + return "{} row {}".format(action, pk_label) async def _resolve_row_and_check_permission(datasette, request, permission): - from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound - - try: - _, _, resource = await _database_and_table_resource_from_request( - datasette, request - ) - except DatabaseNotFound as e: - return False, Response.error([f"Database not found: {e.database_name}"], 404) - - # Check the URL resource before resolving the row, so a denied request - # cannot distinguish an existing primary key from a missing one. - if not await datasette.allowed( - action=permission, - resource=resource, - actor=request.actor, - ): - return False, Response.error(["Permission denied"], 403) + from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound try: resolved = await datasette.resolve_row(request) + except DatabaseNotFound as e: + return False, Response.error( + ["Database not found: {}".format(e.database_name)], 404 + ) except TableNotFound as e: - return False, Response.error([f"Table not found: {e.table}"], 404) + return False, Response.error(["Table not found: {}".format(e.table)], 404) except RowNotFound as e: - return False, Response.error([f"Record not found: {e.pk_values}"], 404) + return False, Response.error(["Record not found: {}".format(e.pk_values)], 404) + + # Ensure user has permission to delete this row + if not await datasette.allowed( + action=permission, + resource=TableResource(database=resolved.db.name, table=resolved.table), + actor=request.actor, + ): + return False, Response.error(["Permission denied"], 403) return True, resolved @@ -798,13 +749,11 @@ class RowDeleteView(BaseView): # Delete table def delete_row(conn): - check_structured_write_table(conn, resolved.table) sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values) try: await resolved.db.execute_write_fn(delete_row, request=request) - except Exception as e: # noqa: BLE001 - # TODO: narrow to expected write errors so Datasette bugs surface as 500s + except Exception as e: return Response.error([str(e)], 400) await self.ds.track_event( @@ -820,7 +769,7 @@ class RowDeleteView(BaseView): table_url = self.ds.urls.table(resolved.db.name, resolved.table) self.ds.add_message( request, - await _row_flash_message(self.ds, request, "Deleted", resolved), + await _row_flash_message(resolved.db, "Deleted", resolved), self.ds.INFO, ) return Response.json({"ok": True, "redirect": str(table_url)}, status=200) @@ -844,7 +793,7 @@ class RowUpdateView(BaseView): try: data = await request.json() except json.JSONDecodeError as e: - return Response.error([f"Invalid JSON: {e}"]) + return Response.error(["Invalid JSON: {}".format(e)]) except PayloadTooLarge as e: return Response.error([str(e)], 413) @@ -881,27 +830,18 @@ class RowUpdateView(BaseView): return Response.error(["Permission denied for alter-table"], 403) def update_row(conn): - check_structured_write_table(conn, resolved.table) sqlite_utils.Database(conn)[resolved.table].update( resolved.pk_values, update, alter=alter ) try: await resolved.db.execute_write_fn(update_row, request=request) - except Exception as e: # noqa: BLE001 - # TODO: narrow to expected write errors so Datasette bugs surface as 500s + except Exception as e: return Response.error([str(e)], 400) result = {"ok": True} returned_row = None - # Only read back and disclose the stored row if the actor is also - # allowed to view this table - update-row alone must not be usable - # to read data the actor cannot otherwise see. - if data.get("return") and await self.ds.allowed( - action="view-table", - resource=TableResource(database=resolved.db.name, table=resolved.table), - actor=request.actor, - ): + if data.get("return"): results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) @@ -918,15 +858,16 @@ class RowUpdateView(BaseView): ) if request.args.get("_message"): + message_row = returned_row + if message_row is None: + results = await resolved.db.execute( + resolved.sql, resolved.params, truncate=True + ) + message_row = results.first() self.ds.add_message( request, await _row_flash_message( - self.ds, - request, - "Updated", - resolved, - row=returned_row, - refresh_row=True, + resolved.db, "Updated", resolved, row=message_row ), self.ds.INFO, ) diff --git a/datasette/views/special.py b/datasette/views/special.py index 72e80316..c13191a1 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,25 +1,23 @@ import json import logging -import secrets -import urllib - -from datasette.events import CreateTokenEvent, LoginEvent, LogoutEvent from datasette.jump import JumpSQL, namespace_sql_params from datasette.plugins import pm +from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource +from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( UNSTABLE_API_MESSAGE, actor_matches_allow, + parse_size_limit, add_cors_headers, await_me_maybe, error_body, - parse_size_limit, - tilde_decode, tilde_encode, + tilde_decode, ) -from datasette.utils.asgi import Forbidden, Response - from .base import BaseView, View +import secrets +import urllib logger = logging.getLogger(__name__) @@ -181,7 +179,9 @@ class AutocompleteDebugView(BaseView): ) context.update( { - "autocomplete_url": f"{self.ds.urls.table(database_name, table_name)}/-/autocomplete", + "autocomplete_url": "{}/-/autocomplete".format( + self.ds.urls.table(database_name, table_name) + ), "label_column": await db.label_column_for_table(table_name), } ) @@ -311,7 +311,6 @@ class AllowedResourcesView(BaseView): has_json_alternate = False async def get(self, request): - await self.ds.ensure_permission(action="view-instance", actor=request.actor) await self.ds.refresh_schemas() # Check if user has permissions-debug (to show sensitive fields) @@ -421,11 +420,8 @@ class AllowedResourcesView(BaseView): row["reason"] = resource.reasons allowed_rows.append(row) - except Exception: # noqa: BLE001 - # Returns empty results if the catalog tables don't exist yet, but - # also swallows the AttributeError raised for instance-level actions - # such as view-instance, which have no resource_class. - # TODO: handle that case explicitly and narrow this to sqlite3.Error + except Exception: + # If catalog tables don't exist yet, return empty results return ( { "ok": True, @@ -527,7 +523,7 @@ class PermissionRulesView(BaseView): from datasette.utils.actions_sql import build_permission_rules_sql - union_sql, union_params, _restriction_sqls = await build_permission_rules_sql( + union_sql, union_params, restriction_sqls = await build_permission_rules_sql( self.ds, actor, action ) await self.ds.refresh_schemas() @@ -604,7 +600,7 @@ class PermissionRulesView(BaseView): async def _check_permission_for_actor(ds, action, parent, child, actor): - """Shared logic for checking and explaining a permission decision.""" + """Shared logic for checking permissions. Returns a dict with check results.""" if action not in ds.actions: return error_body(f"Unknown action: {action}", 404), 404 @@ -633,28 +629,15 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor) - from datasette.utils.actions_sql import explain_permission_for_resource - - explanation = await explain_permission_for_resource( - datasette=ds, - actor=actor, - action=action, - parent=parent, - child=child, - ) - response = { "ok": True, - "unstable": UNSTABLE_API_MESSAGE, "action": action, "allowed": bool(allowed), - "actor": actor, "resource": { "parent": parent, "child": child, "path": _resource_path(parent, child), }, - "explanation": explanation, } if actor and "id" in actor: @@ -672,25 +655,11 @@ class PermissionCheckView(BaseView): as_format = request.url_vars.get("format") if not as_format: - actions = [ - { - "name": action.name, - "description": action.description, - "takes_parent": action.takes_parent, - "takes_child": action.takes_child, - "also_requires": action.also_requires, - } - for action in sorted( - self.ds.actions.values(), key=lambda action: action.name - ) - ] return await self.render( ["debug_check.html"], request, { - "actions": actions, - "actor_json": request.args.get("actor") - or json.dumps(request.actor, indent=2), + "sorted_actions": sorted(self.ds.actions.keys()), "has_debug_permission": True, }, ) @@ -702,18 +671,9 @@ class PermissionCheckView(BaseView): parent = request.args.get("parent") child = request.args.get("child") - actor = request.actor - actor_json = request.args.get("actor") - if actor_json is not None: - try: - actor = json.loads(actor_json) - except json.JSONDecodeError as ex: - return Response.error(f"Invalid actor JSON: {ex}", 400) - if actor is not None and not isinstance(actor, dict): - return Response.error("actor must be a JSON object or null", 400) response, status = await _check_permission_for_actor( - self.ds, action, parent, child, actor + self.ds, action, parent, child, request.actor ) return Response.json(response, status=status) @@ -797,8 +757,6 @@ class CreateTokenView(BaseView): raise Forbidden( "Token authentication cannot be used to create additional tokens" ) - if "_r" in request.actor: - raise Forbidden("Restricted actors cannot create API tokens") async def shared(self, request): self.check_permission(request) @@ -876,11 +834,6 @@ class CreateTokenView(BaseView): else: errors.append("Invalid expire duration unit") - if errors: - context = await self.shared(request) - context["errors"] = errors - return await self.render(["create_token.html"], request, context) - # Are there any restrictions? from datasette.tokens import TokenRestrictions @@ -947,7 +900,7 @@ class ApiExplorerView(BaseView): tables.append({"name": table, "links": table_links}) table_links.append( { - "label": f"Get rows for {table}", + "label": "Get rows for {}".format(table), "method": "GET", "path": self.ds.urls.table(name, table, format="json"), } @@ -967,7 +920,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/insert", "method": "POST", - "label": f"Insert rows into {table}", + "label": "Insert rows into {}".format(table), "json": { "rows": [ { @@ -981,7 +934,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/upsert", "method": "POST", - "label": f"Upsert rows into {table}", + "label": "Upsert rows into {}".format(table), "json": { "rows": [ { @@ -1011,7 +964,7 @@ class ApiExplorerView(BaseView): table_links.append( { "path": self.ds.urls.table(name, table) + "/-/drop", - "label": f"Drop table {table}", + "label": "Drop table {}".format(table), "json": {"confirm": False}, "method": "POST", } @@ -1028,7 +981,7 @@ class ApiExplorerView(BaseView): database_links.append( { "path": self.ds.urls.database(name) + "/-/create", - "label": f"Create table in {name}", + "label": "Create table in {}".format(name), "json": { "table": "new_table", "columns": [ @@ -1269,21 +1222,14 @@ class SchemaBaseView(BaseView): has_json_alternate = False - async def get_database_schema(self, database_name, actor): + async def get_database_schema(self, database_name): """Get schema SQL for a database.""" db = self.ds.databases[database_name] - allowed_tables_page = await self.ds.allowed_resources( - "view-table", actor, parent=database_name - ) - allowed_table_names = { - resource.child async for resource in allowed_tables_page.all() - } result = await db.execute( - "select tbl_name, sql from sqlite_master where sql is not null" - ) - return ";\n".join( - row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names + "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null" ) + row = result.first() + return row["schema"] if row and row["schema"] else "" def format_json_response(self, data): """Format data as JSON response with CORS headers if needed.""" @@ -1345,7 +1291,7 @@ class InstanceSchemaView(SchemaBaseView): # Get schema for each database schemas = [] for database_name in allowed_databases: - schema = await self.get_database_schema(database_name, request.actor) + schema = await self.get_database_schema(database_name) schemas.append({"database": database_name, "schema": schema}) if format_ == "json": @@ -1386,7 +1332,7 @@ class DatabaseSchemaView(SchemaBaseView): if database_name not in self.ds.databases: return self.format_error_response("Database not found", format_) - schema = await self.get_database_schema(database_name, request.actor) + schema = await self.get_database_schema(database_name) if format_ == "json": return self.format_json_response( @@ -1425,8 +1371,7 @@ class TableSchemaView(SchemaBaseView): # Get schema for the table db = self.ds.databases[database_name] result = await db.execute( - "select sql from sqlite_master where name = ? " - "and type in ('table', 'view') and sql is not null", + "select sql from sqlite_master where name = ? and sql is not null", [table_name], ) row = result.first() diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 03bd9b29..d64f37d2 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -124,7 +124,7 @@ class QueryListView(BaseView): pairs.append(("_next", page.next)) next_url = self.ds.absolute_url( request, - f"{request.path}?{urlencode(pairs)}", + "{}?{}".format(request.path, urlencode(pairs)), ) current_filters = { @@ -279,7 +279,7 @@ class QueryCreateView(BaseView): ), ) response.status = status - return _block_framing(response) + return response async def get(self, request): db = await self.ds.resolve_database(request) @@ -415,7 +415,7 @@ class QueryDefinitionView(BaseView): query_name = tilde_decode(request.url_vars["query"]) query = await self.ds.get_query(db.name, query_name) if query is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="view-query", resource=QueryResource(db.name, query_name), @@ -439,7 +439,7 @@ class QueryUpdateView(BaseView): query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), @@ -527,12 +527,12 @@ class QueryEditView(BaseView): ), ) response.status = status - return _block_framing(response) + return response async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="update-query", resource=QueryResource(db.name, query_name), @@ -545,7 +545,7 @@ class QueryEditView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), @@ -629,7 +629,7 @@ class QueryDeleteView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="delete-query", resource=QueryResource(db.name, query_name), @@ -639,23 +639,21 @@ class QueryDeleteView(BaseView): return Response.error( ["Trusted queries cannot be deleted using the API"], 403 ) - return _block_framing( - await self.render( - ["query_delete.html"], - request, - { - "database": db.name, - "database_color": db.color, - "query": stored_query_to_dict(existing), - "query_url": self.ds.urls.table(db.name, query_name), - }, - ) + return await self.render( + ["query_delete.html"], + request, + { + "database": db.name, + "database_color": db.color, + "query": stored_query_to_dict(existing), + "query_url": self.ds.urls.table(db.name, query_name), + }, ) async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="delete-query", resource=QueryResource(db.name, query_name), @@ -667,13 +665,13 @@ class QueryDeleteView(BaseView): ["Trusted queries cannot be deleted using the API"], 403 ) - _data, is_json = await _json_or_form_payload(request) + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) if is_json: return Response.json({"ok": True}) self.ds.add_message( request, - f"Query “{existing.title or query_name}” deleted", + "Query “{}” deleted".format(existing.title or query_name), self.ds.INFO, ) return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/datasette/views/table.py b/datasette/views/table.py index 26ce2acd..f7edd744 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,54 +1,50 @@ import asyncio import itertools import json -import time import urllib import urllib.parse -from dataclasses import dataclass, field import markupsafe -import sqlite_utils -from datasette import tracer from datasette.column_types import SQLiteType -from datasette.database import QueryInterrupted +from datasette.extras import extra_names_from_request +from datasette.plugins import pm from datasette.events import ( AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, ) -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.filters import Filters -from datasette.plugins import pm +from datasette.database import QueryInterrupted +from datasette import tracer from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( - CustomJSONEncoder, - CustomRow, - InvalidSql, - WriteJsonValueError, add_cors_headers, - append_querystring, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, + CustomRow, + append_querystring, compound_keys_after_sql, decode_write_json_rows, + format_bytes, + make_slot_function, + tilde_encode, escape_sqlite, filters_should_redirect, - format_bytes, is_url, - make_slot_function, path_from_row_pks, path_with_added_args, path_with_format, path_with_removed_args, path_with_replaced_args, - sqlite3, - tilde_encode, to_css_class, truncate_url, urlsafe_components, value_as_boolean, + InvalidSql, + WriteJsonValueError, + sqlite3, ) from datasette.utils.asgi import ( BadRequest, @@ -58,8 +54,11 @@ from datasette.utils.asgi import ( Request, Response, ) -from datasette.utils.sqlite import check_structured_write_table +from datasette.filters import Filters +import sqlite_utils +from dataclasses import dataclass, field +from datasette.extras import ExtraScope from . import Context, from_extra from .base import BaseView, DatasetteError, stream_csv from .database import QueryView @@ -537,7 +536,7 @@ async def _table_insert_ui( columns.append(column_data) data = { - "path": f"{datasette.urls.table(database_name, table_name)}/-/insert", + "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, "bulkColumns": bulk_columns, @@ -545,8 +544,8 @@ async def _table_insert_ui( "maxInsertRows": datasette.setting("max_insert_rows"), } if can_update: - data["upsertPath"] = ( - f"{datasette.urls.table(database_name, table_name)}/-/upsert" + data["upsertPath"] = "{}/-/upsert".format( + datasette.urls.table(database_name, table_name) ) return data @@ -605,7 +604,7 @@ async def _table_alter_ui( columns.append(column_data) data = { - "path": f"{datasette.urls.table(database_name, table_name)}/-/alter", + "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, "primaryKeys": pks, @@ -631,7 +630,9 @@ async def _table_alter_ui( actor=request.actor, ) if can_drop_table: - data["dropPath"] = f"{datasette.urls.table(database_name, table_name)}/-/drop" + data["dropPath"] = "{}/-/drop".format( + datasette.urls.table(database_name, table_name) + ) return data @@ -672,7 +673,7 @@ async def display_columns_and_rows( } pks = await db.primary_keys(table_name) pks_for_display = pks - if not pks_for_display and not await db.view_exists(table_name): + if not pks_for_display: pks_for_display = ["rowid"] label_column = None if link_column: @@ -727,10 +728,12 @@ async def display_columns_and_rows( row_label = row_label_from_label_column(row, label_column) row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = f"{pk_path} {row_label}" + row_action_label = "{} {}".format(pk_path, row_label) table_path = datasette.urls.table(database_name, table_name) - row_link = ( - f'{markupsafe.escape(pk_path)!s}' + row_link = '{flat_pks}'.format( + table_path=table_path, + flat_pks=str(markupsafe.escape(pk_path)), + flat_pks_quoted=row_path, ) edit_icon = ( '
      - -
      -
      - This dialog uses Datasette's shared styles and keyboard behavior. -
      -
      - -
      `; - - const closeButton = dialog.querySelector("button"); - closeButton.addEventListener("click", () => { - modal.requestClose("cancel"); - }); - openButton.addEventListener("click", () => { - modal.show({ returnFocusTo: openButton, initialFocus: closeButton }); - }); - - document.body.append(modal); - document.querySelector("section.content").append(openButton); - }); - -Opening and closing -~~~~~~~~~~~~~~~~~~~ - -``modal.show(options)`` - Opens the native dialog using ``showModal()``. ``options`` is an optional object with these optional properties: - - - ``returnFocusTo`` (DOM element): Focus returns to this element when the dialog closes. Defaults to the element with keyboard focus immediately before the dialog opens. - - ``initialFocus`` (DOM element or function): An element inside the dialog whose ``focus()`` method will be called, or a function called with no arguments that moves focus itself. - - Use this to focus on an input field when the dialog opens. - -``modal.close(options)`` - Closes the dialog directly. ``options`` is an optional object with one optional property: - - - ``restoreFocus`` (boolean): Whether closing returns focus to the element recorded by ``show()``. Defaults to ``true``. - -``modal.requestClose(source)`` - Alternative to ``.close()`` that requests dismissal through the busy-state and ``beforeClose`` guards described below. Returns ``true`` if it closes the dialog, or ``false`` if the dialog is already closed or a guard prevents dismissal. Close and Cancel buttons should use this method. - - ``source`` is an optional string that is passed to ``beforeClose`` and identifies what requested dismissal. Datasette supplies ``"escape"`` for the Escape key or a native cancel event and ``"backdrop"`` for a click outside the dialog. ``source`` defaults to ``"cancel"``. - -Listen for the native dialog's ``close`` event to clean up resources such as pending requests or custom fields: - -.. code-block:: javascript - - modal.dialog.addEventListener("close", () => { - // Clean up content-specific resources here. - }); - -If the dialog is no longer needed, remove the wrapper with ``modal.remove()``. - -Dismissal guards and busy state -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can set ``modal.beforeClose`` to a synchronous function that receives the ``source`` string described above and returns ``false`` in order to keep the dialog open. - -Use ``source`` to decide what to do. This example prompts the user to ask if they want to discard unsaved changes - for example if they click outside the modal or hit Escape - but doesn't prompt them if they clicked a button like the Close one above that sets the ``source`` string to ``cancel``. - -.. code-block:: javascript - - modal.beforeClose = (source) => { - if (source === "cancel") return true; - return confirm("Discard unsaved changes?"); - }; - - -Set ``modal.busy = true`` while saving to prevent user dismissal. While busy, ``requestClose()`` returns ``false`` without calling ``beforeClose``. - -If an operation fails, set ``modal.busy = false`` so the user can retry or close the dialog. A successful operation can call ``modal.close()`` even while busy. - -.. _javascript_plugins_modal_classes: - -Shared CSS classes -~~~~~~~~~~~~~~~~~~ - -The classes in the example above provide built-in styling. The following classes can be used by your modal: - -``datasette-modal`` - Added automatically to the native ```` when the wrapper is connected to the page. Provides the dialog's sizing, background, rounded corners, shadow, backdrop and animations. - -``modal-header`` - Adds padding, a bottom border and a horizontal layout for the title and optional metadata. - -``modal-title`` - Sets the title's font size, weight and color. Use ``aria-labelledby`` to associate the title with the dialog. - -``modal-meta`` - Styles optional metadata, such as a selected-item count, as small monospace text with a rounded background. - -``modal-body`` - Adds padding and makes overflowing content scroll while the header and footer remain visible. Sets ``min-height: 0``, ``overflow: auto`` and ``padding: 16px 24px 24px``. - -``modal-footer`` - Adds padding, a top border and a background to the action area. Arranges its contents horizontally, with buttons aligned to the right. - -``footer-info`` - Styles supporting text in the footer and lets it fill the space before the action buttons. - -``modal-btn`` - Provides base button styling, including padding, rounded corners, font and disabled appearance. Use it together with ``modal-btn-primary`` or ``modal-btn-ghost``. - -``modal-btn-primary`` - Gives a button an accent-colored background and white text, suitable for a primary action such as Save. - -``modal-btn-ghost`` - Gives a button a transparent background, muted text and a border, suitable for a secondary action such as Close or Cancel. - -These button classes are also used by Datasette's built-in dialogs. - -You can customize layout and sizing without adding extra classes. For example, this CSS uses the dialog's existing ID to widen it while keeping it inside the viewport: - -.. code-block:: css - - dialog#my-plugin-dialog { - width: min(720px, calc(100vw - 32px)); - } - .. _javascript_datasette_manager_selectors: Selectors diff --git a/docs/json_api.rst b/docs/json_api.rst index 73212e70..a96fd73d 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -48,7 +48,6 @@ Some JSON endpoints are **exempt** from this promise: debug playground. - Debug and support endpoints are documented so you can use them, but their JSON shapes are not frozen: :ref:`/-/threads `, - :ref:`/-/tasks `, :ref:`/-/actions `, the :ref:`permission debug endpoints ` (``/-/allowed``, ``/-/rules``, ``/-/check``) and the @@ -1327,23 +1326,6 @@ The following extras are available for arbitrary SQL query responses and stored, .. [[[end]]] -.. _TableCountView: - -Counting all matching rows --------------------------- - -``POST //
      /-/count`` returns an exact count of the rows matching the table's query string filters:: - - POST /fixtures/facetable/-/count?state=CA - - {"ok": true, "count": 10} - -The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count. - -This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters. - -Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error. - .. _TableAutocompleteView: Table autocomplete @@ -1679,8 +1661,6 @@ The request body is always parsed as JSON, regardless of the request's ``Content The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data. -Structured inserts, upserts, updates and deletes only support ordinary SQLite tables. Virtual tables and their internal shadow tables are rejected, including when adding rows to an existing table through the create-table API. Writes to ordinary content tables can still update full-text search indexes through configured triggers. - .. _ExecuteWriteView: Executing write SQL diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 9a4dba23..422e67f4 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -46,7 +46,7 @@ def table_extras(cog): cog.out("\n") for scope, heading, intro, classes in classes_by_scope: cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) - cog.out(f"{intro}\n\n") + cog.out("{}\n\n".format(intro)) for cls in classes: examples = _examples_for_scope(cls, scope) description = cls.description or "" @@ -58,16 +58,16 @@ def table_extras(cog): if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out(f"``{cls.key()}``\n") - cog.out(f" {description}\n\n") + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) for example in examples: if example.path: value = live_examples[(example.path, example.key or cls.key())] - cog.out(f" ``GET {example.path}``\n\n") + cog.out(" ``GET {}``\n\n".format(example.path)) else: value = example.value if example.note: - cog.out(f" {example.note}\n\n") + cog.out(" {}\n\n".format(example.note)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) cog.out("\n\n") @@ -139,7 +139,7 @@ async def _fetch_live_examples(scoped_classes): response = await datasette.client.get(example.path) assert response.status_code == 200, example.path data = response.json() - assert key in data, f"{key} missing from {example.path}" + assert key in data, "{} missing from {}".format(key, example.path) examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 1bf17f8e..031b3ddd 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,8 +1,7 @@ import json import textwrap - -from ruamel.yaml import YAML from yaml import safe_dump +from ruamel.yaml import YAML def metadata_example(cog, data=None, yaml=None): @@ -34,10 +33,10 @@ def config_example( else: data = input output_yaml = safe_dump(input, sort_keys=False) - cog.out(f"\n.. tab:: {yaml_title}\n\n") + cog.out("\n.. tab:: {}\n\n".format(yaml_title)) cog.out(" .. code-block:: yaml\n\n") cog.out(textwrap.indent(output_yaml, " ")) - cog.out(f"\n\n.. tab:: {json_title}\n\n") + cog.out("\n\n.. tab:: {}\n\n".format(json_title)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") @@ -45,10 +44,8 @@ def config_example( def internal_schema(cog): import asyncio - - from sqlite_utils import Database - from datasette.app import Datasette + from sqlite_utils import Database ds = Datasette() db = ds.get_internal_database() diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 3efff7a4..049cb292 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -217,7 +217,7 @@ Extra template variables that should be made available in the rendered template ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` -This hook supports the following return values: +This hook can return one of three different types: Dictionary If you return a dictionary its keys and values will be merged into the template context. @@ -228,9 +228,6 @@ Function that returns a dictionary Function that returns an awaitable function that returns a dictionary You can also return a function which returns an awaitable function which returns a dictionary. -``None`` - The hook itself, or a function or awaitable it returns, can return ``None`` when no extra variables are needed. Variables returned by other plugins are still included. - Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template. .. warning:: @@ -257,6 +254,8 @@ This example returns an awaitable function which adds a list of ``hidden_table_n return { "hidden_table_names": await db.hidden_table_names() } + else: + return {} return hidden_table_names @@ -496,10 +495,10 @@ Lets you customize the display of values within table cells in the HTML table vi The name of the column being rendered ``table`` - string or None - The name of the table or view - or ``None`` if this is a custom SQL query + The name of the table - or ``None`` if this is a custom SQL query ``pks`` - list of strings - The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views, this will be an empty list ``[]``. + The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views (where ``table`` is ``None``), this will be an empty list ``[]``. ``database`` - string The name of the database @@ -1108,7 +1107,7 @@ Return an `ASGI `__ middleware wrapper function th This is a very powerful hook. You can use it to manipulate the entire Datasette response, or even to configure new URL routes that will be handled by your own custom code. -You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette `__. +You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette `__. This example plugin adds a ``x-databases`` HTTP header listing the currently attached databases: @@ -1158,7 +1157,7 @@ Examples: `datasette-cors `__, `dat startup(datasette) ------------------ -This hook fires when the Datasette application server first starts up. It runs on the same event loop that goes on to serve requests, so it is safe to create loop-bound primitives and register background work here — see :ref:`datasette_lifecycle` for the full guarantee and the three ways startup can be triggered. +This hook fires when the Datasette application server first starts up. Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails: @@ -1196,7 +1195,6 @@ Potential use-cases: * Create database tables that a plugin needs on startup * Validate the configuration for a plugin on startup, and raise an error if it is invalid * Raise a ``datasette.utils.StartupError("message")`` exception to prevent Datasette from starting and display that message to the user. -* Register supervised long-lived background work using :ref:`datasette_add_background_task`, which core launches once every plugin's ``startup()`` hook has finished. .. note:: @@ -1213,31 +1211,6 @@ Potential use-cases: Examples: `datasette-saved-queries `__, `datasette-init `__ -.. _plugin_hook_shutdown: - -shutdown(datasette) -------------------- - -This hook fires once, when the Datasette application server is shutting down gracefully - triggered by the ASGI ``lifespan.shutdown`` event, which includes pressing Ctrl-C or sending ``SIGTERM`` to a ``datasette serve`` process. It is not called on a hard kill (``SIGKILL``), since there is no opportunity to run any code in that case. - -Like ``startup()``, this can be a regular function or it can return an async function to be awaited. - -It runs before Datasette cancels any background tasks it is supervising (see :ref:`datasette_add_background_task`) and before it closes its database connections, so you can use it to tell your plugin's own background work to stop gracefully while a database connection is still available to write out any final state. See :ref:`datasette_lifecycle` for exactly where this fits into the full startup-to-shutdown sequence: - -.. code-block:: python - - @hookimpl - def shutdown(datasette): - async def inner(): - db = datasette.get_database() - await db.execute_write( - "insert into shutdown_log (at) values (datetime('now'))" - ) - - return inner - -If your ``shutdown()`` hook raises an exception it will be logged but not re-raised, so one plugin's broken shutdown code cannot prevent other plugins - or Datasette itself - from finishing their own teardown. - .. _plugin_hook_actor_from_request: actor_from_request(datasette, request) diff --git a/docs/plugins.rst b/docs/plugins.rst index 296ef55d..d32a9fe6 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -261,15 +261,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "permission_resources_sql" ] }, - { - "name": "datasette.default_permissions.sqlite_statistics", - "static": false, - "templates": false, - "version": null, - "hooks": [ - "permission_resources_sql" - ] - }, { "name": "datasette.default_permissions.tokens", "static": false, diff --git a/docs/settings.rst b/docs/settings.rst index f3e6636d..9c114e4a 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -67,21 +67,10 @@ The following options can be set using ``--setting name value``, or by storing t default_allow_sql ~~~~~~~~~~~~~~~~~ -.. [[[cog - from settings_doc import setting_default - setting_default(cog, "default_allow_sql") -.. ]]] - -Default: ``on`` - -.. [[[end]]] - Should users be able to execute arbitrary SQL queries by default? Setting this to ``off`` causes permission checks for :ref:`actions_execute_sql` to fail by default. -This setting controls the ability to submit arbitrary SQL. It does not disable structured table-browsing features that use SQL generated by Datasette, such as sorting, column filters and :ref:`facets`. Use :ref:`setting_allow_facet` to control whether users can request facets. - :: datasette mydatabase.db --setting default_allow_sql off @@ -93,14 +82,6 @@ Another way to achieve this is to add ``"allow_sql": false`` to your ``datasette default_page_size ~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "default_page_size") -.. ]]] - -Default: ``100`` - -.. [[[end]]] - The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` query string parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--setting`` like so:: datasette mydatabase.db --setting default_page_size 50 @@ -110,15 +91,7 @@ The default number of rows returned by the table page. You can over-ride this on sql_time_limit_ms ~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "sql_time_limit_ms") -.. ]]] - -Default: ``1000`` - -.. [[[end]]] - -Time limit for SQL queries, in milliseconds. If a query takes longer than this to run Datasette will terminate the query and return an error. +By default, queries have a time limit of one second. If a query takes longer than this to run Datasette will terminate the query and return an error. If this time limit is too short for you, you can customize it using the ``sql_time_limit_ms`` limit - for example, to increase it to 3.5 seconds:: @@ -135,15 +108,7 @@ This would set the time limit to 100ms for that specific query. This feature is max_returned_rows ~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "max_returned_rows") -.. ]]] - -Default: ``1000`` - -.. [[[end]]] - -The maximum number of rows Datasette returns at a time. If you execute a query that exceeds this limit, Datasette will truncate the result set and include a warning. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more rows. +Datasette returns a maximum of 1,000 rows of data at a time. If you execute a query that returns more than 1,000 rows, Datasette will return the first 1,000 and include a warning that the result set has been truncated. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more than 1,000 rows. You can increase or decrease this limit like so:: @@ -154,15 +119,7 @@ You can increase or decrease this limit like so:: max_insert_rows ~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "max_insert_rows") -.. ]]] - -Default: ``100`` - -.. [[[end]]] - -Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. +Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100. You can increase or decrease this limit like so:: @@ -173,15 +130,7 @@ You can increase or decrease this limit like so:: max_post_body_bytes ~~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "max_post_body_bytes") -.. ]]] - -Default: ``2097152`` - -.. [[[end]]] - -Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API `. Requests with larger bodies are rejected with an HTTP 413 error. +Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API `. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB). This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over. @@ -198,15 +147,7 @@ Set it to 0 to disable the limit entirely:: num_sql_threads ~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "num_sql_threads") -.. ]]] - -Default: ``3`` - -.. [[[end]]] - -Maximum number of threads in the thread pool Datasette uses to execute SQLite queries. +Maximum number of threads in the thread pool Datasette uses to execute SQLite queries. Defaults to 3. :: @@ -219,17 +160,9 @@ Setting this to 0 turns off threaded SQL queries entirely - useful for environme allow_facet ~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "allow_facet") -.. ]]] - -Default: ``on`` - -.. [[[end]]] - Allow users to specify columns they would like to facet on using the ``?_facet=COLNAME`` URL parameter to the table view. -If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table. +This is enabled by default. If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table. Here's how to disable this feature:: @@ -240,15 +173,7 @@ Here's how to disable this feature:: default_facet_size ~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "default_facet_size") -.. ]]] - -Default: ``30`` - -.. [[[end]]] - -The default number of unique rows returned by :ref:`facets`. You can customize it like this:: +The default number of unique rows returned by :ref:`facets` is 30. You can customize it like this:: datasette mydatabase.db --setting default_facet_size 50 @@ -257,15 +182,7 @@ The default number of unique rows returned by :ref:`facets`. You can customize i facet_time_limit_ms ~~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "facet_time_limit_ms") -.. ]]] - -Default: ``200`` - -.. [[[end]]] - -The time limit in milliseconds Datasette allows for calculating a facet. You can customize it like this:: +This is the time limit Datasette allows for calculating a facet, which defaults to 200ms:: datasette mydatabase.db --setting facet_time_limit_ms 1000 @@ -274,15 +191,7 @@ The time limit in milliseconds Datasette allows for calculating a facet. You can facet_suggest_time_limit_ms ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "facet_suggest_time_limit_ms") -.. ]]] - -Default: ``50`` - -.. [[[end]]] - -When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. This time limit, in milliseconds, applies separately to each query. If the time limit is exceeded the column will not be suggested as a facet. +When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. The default for this time limit is 50ms to account for the fact that it needs to run once for every column. If the time limit is exceeded the column will not be suggested as a facet. You can increase this time limit like so:: @@ -293,15 +202,7 @@ You can increase this time limit like so:: suggest_facets ~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "suggest_facets") -.. ]]] - -Default: ``on`` - -.. [[[end]]] - -Should Datasette calculate suggested facets? Turn this off like so:: +Should Datasette calculate suggested facets? On by default, turn this off like so:: datasette mydatabase.db --setting suggest_facets off @@ -310,15 +211,7 @@ Should Datasette calculate suggested facets? Turn this off like so:: allow_download ~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "allow_download") -.. ]]] - -Default: ``on`` - -.. [[[end]]] - -Should users be able to download the original SQLite database using a link on the database index page? Databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following:: +Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default. However, databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following:: datasette mydatabase.db --setting allow_download off @@ -327,17 +220,9 @@ Should users be able to download the original SQLite database using a link on th allow_signed_tokens ~~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "allow_signed_tokens") -.. ]]] - -Default: ``on`` - -.. [[[end]]] - Should users be able to create signed API tokens to access Datasette? -Use the following to turn it off:: +This is turned on by default. Use the following to turn it off:: datasette mydatabase.db --setting allow_signed_tokens off @@ -348,17 +233,9 @@ Turning this setting off will disable the ``/-/create-token`` page, :ref:`descri max_signed_tokens_ttl ~~~~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "max_signed_tokens_ttl") -.. ]]] - -Default: ``0`` - -.. [[[end]]] - Maximum allowed expiry time for signed API tokens created by users. -A value of ``0`` means no limit - tokens can be created that will never expire. +Defaults to ``0`` which means no limit - tokens can be created that will never expire. Set this to a value in seconds to limit the maximum expiry time. For example, to set that limit to 24 hours you would use:: @@ -371,36 +248,18 @@ This setting is enforced when incoming tokens are processed. default_cache_ttl ~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "default_cache_ttl") -.. ]]] - -Default: ``5`` - -.. [[[end]]] - -Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely. +Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely. Defaults to 5 seconds. :: datasette mydatabase.db --setting default_cache_ttl 60 -Dynamic responses for authenticated actors, requests with cookies or an ``Authorization`` header, and responses that set cookies use ``Cache-Control: private, no-store``. This takes precedence over ``default_cache_ttl`` and ``?_ttl=``, even when cache headers are otherwise disabled. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``. Static assets retain their own cache policy. - .. _setting_cache_size_kb: cache_size_kb ~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "cache_size_kb") -.. ]]] - -Default: ``0`` - -.. [[[end]]] - -Sets the amount of memory SQLite uses for its `per-connection cache `_, in KB. Set this to ``0`` to use SQLite's default cache size. +Sets the amount of memory SQLite uses for its `per-connection cache `_, in KB. :: @@ -411,17 +270,9 @@ Sets the amount of memory SQLite uses for its `per-connection cache ` where an entire table (potentially hundreds of thousands of rows) can be exported as a single CSV -file. You can turn it off like this: +file. This is turned on by default - you can turn it off like this: :: @@ -432,16 +283,8 @@ file. You can turn it off like this: max_csv_mb ~~~~~~~~~~ -.. [[[cog - setting_default(cog, "max_csv_mb") -.. ]]] - -Default: ``100`` - -.. [[[end]]] - -The maximum size of CSV that can be exported, in megabytes. -You can disable the limit entirely by setting this to 0: +The maximum size of CSV that can be exported, in megabytes. Defaults to 100MB. +You can disable the limit entirely by settings this to 0: :: @@ -452,14 +295,6 @@ You can disable the limit entirely by setting this to 0: truncate_cells_html ~~~~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "truncate_cells_html") -.. ]]] - -Default: ``2048`` - -.. [[[end]]] - In the HTML table view, truncate any strings that are longer than this value. The full value will still be available in CSV, JSON and on the individual row HTML page. Set this to 0 to disable truncation. @@ -473,14 +308,6 @@ HTML page. Set this to 0 to disable truncation. force_https_urls ~~~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "force_https_urls") -.. ]]] - -Default: ``off`` - -.. [[[end]]] - Forces self-referential URLs in the JSON output to always use the ``https://`` protocol. This is useful for cases where the application itself is hosted using HTTP but is served to the outside world via a proxy that enables HTTPS. @@ -494,14 +321,6 @@ HTTP but is served to the outside world via a proxy that enables HTTPS. template_debug ~~~~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "template_debug") -.. ]]] - -Default: ``off`` - -.. [[[end]]] - This setting enables template context debug mode, which is useful to help understand what variables are available to custom templates when you are writing them. Enable it like this:: @@ -521,14 +340,6 @@ Some examples: trace_debug ~~~~~~~~~~~ -.. [[[cog - setting_default(cog, "trace_debug") -.. ]]] - -Default: ``off`` - -.. [[[end]]] - This setting enables appending ``?_trace=1`` to any page in order to see the SQL queries and other trace information that was used to generate that page. Enable it like this:: @@ -547,14 +358,6 @@ See :ref:`internals_tracer` for details on how to hook into this mechanism as a base_url ~~~~~~~~ -.. [[[cog - setting_default(cog, "base_url") -.. ]]] - -Default: ``/`` - -.. [[[end]]] - If you are running Datasette behind a proxy, it may be useful to change the root path used for the Datasette instance. For example, if you are sending traffic from ``https://www.example.com/tools/datasette/`` through to a proxied Datasette instance you may wish Datasette to use ``/tools/datasette/`` as its root URL. diff --git a/docs/settings_doc.py b/docs/settings_doc.py deleted file mode 100644 index e041f333..00000000 --- a/docs/settings_doc.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Cog helper for documenting setting defaults from Datasette's registry.""" - - -def setting_default(cog, name): - from datasette.app import DEFAULT_SETTINGS - - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - default = "on" if default else "off" - cog.out(f"\nDefault: ``{default}``\n\n") diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index 95b8a366..a5f4fb6f 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -21,12 +21,14 @@ def template_context(cog): ), ) for name, doc in TEMPLATE_BASE_CONTEXT.items(): - cog.out(f"``{name}``\n") - cog.out(f" {doc}\n\n") + cog.out("``{}``\n".format(name)) + cog.out(" {}\n\n".format(doc)) for klass in PAGES.values(): title = "{} page".format(klass.__name__.removesuffix("Context")) - intro = f"{klass.__doc__} Rendered using the ``{klass.documented_template}`` template." + intro = "{} Rendered using the ``{}`` template.".format( + klass.__doc__, klass.documented_template + ) _section(cog, title, intro) if klass.extras_scope is not None: cog.out( @@ -34,10 +36,10 @@ def template_context(cog): "` for this page.\n\n" ) for f in sorted(klass.documented_fields(), key=lambda f: f.name): - cog.out(f"``{f.name}`` - ``{f.type_name}``\n") - cog.out(f" {f.help}\n\n") + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) def _section(cog, title, intro): cog.out("{}\n{}\n\n".format(title, "-" * len(title))) - cog.out(f"{intro}\n\n") + cog.out("{}\n\n".format(intro)) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index cf0241ac..15891963 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl ) -This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 `__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. +This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX `__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. This test also uses the `pytest-asyncio `__ package to add support for ``async def`` test functions running under pytest. @@ -78,19 +78,9 @@ Creating a ``Datasette()`` instance like this as useful shortcut in tests, but t datasette = Datasette(memory=True) await datasette.invoke_startup() -This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. It runs on the same event loop that runs your test, matching the guarantee described in :ref:`datasette_lifecycle`. +This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. -If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request, via the first-request fallback described in :ref:`datasette_lifecycle`. - -If your plugin also registers work with :ref:`datasette_add_background_task` (typically from a ``startup`` hook) and your test needs that work to actually run, call ``await datasette.start_background_tasks()`` as well - ``invoke_startup()`` alone only runs ``startup`` hooks, it does not launch anything they registered: - -.. code-block:: python - - datasette = Datasette(memory=True) - await datasette.start_background_tasks() - # Any tasks registered by a startup() hook are now running - -A request made through ``datasette.client`` arms both startup and background-task launch automatically, since they're both part of the same first-request fallback - ``start_background_tasks()`` is for tests that need tasks running without making an HTTP request first. +If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. .. _testing_plugins_datasette_fixtures_database: @@ -164,7 +154,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini` Using datasette.client in tests ------------------------------- -The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. +The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. A simple test looks like this: @@ -283,22 +273,22 @@ If you want to create that test database repeatedly for every individual test fu .. _testing_plugins_pytest_httpx: -Testing outbound HTTP calls with pytest-httpx2 ----------------------------------------------- +Testing outbound HTTP calls with pytest-httpx +--------------------------------------------- If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests. -The `pytest-httpx2 `__ package provides a ``httpx2_mock`` fixture, built on `respx `__, for mocking outbound calls made using HTTPX2. +The `pytest-httpx `__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally. -Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock. +To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture. -As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content: +As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content: .. code-block:: python from datasette import hookimpl from datasette.utils.asgi import Response - import httpx2 + import httpx @hookimpl @@ -316,18 +306,27 @@ As an example, here's a very simple plugin which executes an HTTP request and re """) vars = await request.post_vars() url = vars["url"] - return Response.text(httpx2.get(url).text) + return Response.text(httpx.get(url).text) -Here's a test for that plugin that mocks the HTTPX2 outbound request: +Here's a test for that plugin that mocks the HTTPX outbound request: .. code-block:: python from datasette.app import Datasette + import pytest - async def test_outbound_http_call(httpx2_mock): - httpx2_mock.get("https://www.example.com/").respond( - text="Hello world" + @pytest.fixture + def non_mocked_hosts(): + # This ensures httpx-mock will not affect Datasette's own + # httpx calls made in the tests by datasette.client: + return ["localhost"] + + + async def test_outbound_http_call(httpx_mock): + httpx_mock.add_response( + url="https://www.example.com/", + text="Hello world", ) datasette = Datasette([], memory=True) response = await datasette.client.post( @@ -336,13 +335,11 @@ Here's a test for that plugin that mocks the HTTPX2 outbound request: ) assert response.text == "Hello world" - outbound_request = httpx2_mock.calls.last.request + outbound_request = httpx_mock.get_request() assert ( outbound_request.url == "https://www.example.com/" ) -If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx `__. - .. _testing_plugins_register_in_test: Registering a plugin for the duration of a test diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index bbefe344..d1e5e75a 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -203,40 +203,6 @@ Templates should be bundled for distribution using the same ``package_data`` mec You can also use wildcards here such as ``templates/*.html``. See `datasette-edit-schema `__ for an example of this pattern. -.. _writing_plugins_custom_templates_breadcrumbs: - -Adding breadcrumbs -~~~~~~~~~~~~~~~~~~ - -Plugin templates that extend ``base.html`` can use the ``crumbs.nav()`` macro to display breadcrumb links back to the Datasette homepage, and optionally to a database and a table. Override the ``crumbs`` block to specify which links to include: - -.. code-block:: html+jinja - - {% extends "base.html" %} - - {% block title %}Manage {{ table }}{% endblock %} - - {% block crumbs %} - - {{ crumbs.nav(request=request, database=database, table=table) }} - - {{ crumbs.nav(request=request, database=database) }} - {% endblock %} - - {% block content %} -

      Manage {{ table }}

      - {% endblock %} - -The macro accepts these arguments: - -* ``request``: the current request, used to check the actor's permissions. -* ``database``: an optional database name, as a string -* ``table``: an optional table name, as a string. If you pass ``table``, you must also pass ``database``. - -For a database-level plugin page, use ``{{ crumbs.nav(request=request, database=database) }}``. For a page with just a homepage link, use ``{{ crumbs.nav(request=request) }}``, which is also the default provided by ``base.html`` if you do not override the block. - -The table-level example renders links in the form ``home / database / table``. Each link is only included if the current actor has permission to view that resource. - .. _writing_plugins_configuration: Writing plugins that accept configuration diff --git a/pyproject.toml b/pyproject.toml index 97acbd6b..70496cdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,9 +28,9 @@ dependencies = [ "click-default-group>=1.2.3", "Jinja2>=2.10.3", "hupper>=1.9", - "httpx2>=2.0", + "httpx>=0.20,<1.0", "pluggy>=1.0", - "uvicorn>=0.29", + "uvicorn>=0.11", "aiofiles>=0.4", "PyYAML>=5.3", "mergedeep>=1.1.1", @@ -69,7 +69,7 @@ dev = [ "trustme>=0.7", "cogapp>=3.3.0", "multipart-form-data-conformance==0.1a0", - "ruff>=0.16.0", + "ruff>=0.9", # docs "Sphinx==7.4.7", "furo==2025.9.25", @@ -102,5 +102,9 @@ datasette = ["templates/*.html"] [tool.setuptools.dynamic] version = {attr = "datasette.version.__version__"} +[tool.ruff] +line-length = 160 +select = ["E", "F", "W"] + [tool.uv] package = true diff --git a/pytest.ini b/pytest.ini index 590054de..75de6925 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,4 @@ [pytest] -addopts = --ignore=ignored filterwarnings= # https://github.com/pallets/jinja/issues/927 ignore:Using or importing the ABCs::jinja2 diff --git a/ruff.toml b/ruff.toml index 3c4345bf..74447a8c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,7 +1,2 @@ line-length = 160 -target-version = "py310" - -[lint.flake8-bugbear] -# from_extra() returns a dataclasses.field(), so it is safe as a dataclass -# default - ruff cannot see through the wrapper (RUF009) -extend-immutable-calls = ["datasette.views.from_extra"] \ No newline at end of file +target-version = "py310" \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index e90b5ab6..7ec03146 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,15 @@ +import httpx import importlib.metadata import os import pathlib +import pytest +import pytest_asyncio import re -import socket import subprocess import sys import tempfile import time from dataclasses import dataclass - -import httpx2 -import pytest -import pytest_asyncio - from datasette import Event, hookimpl try: @@ -33,29 +30,15 @@ UNDOCUMENTED_PERMISSIONS = { } -def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs): +def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): start = time.time() while time.time() - start < timeout: - # If the server died there is no point waiting out the timeout - fail - # now, with its output, instead of after `timeout` seconds of silence - if process is not None and process.poll() is not None: - raise AssertionError( - "Server exited early with returncode {}\n{}".format( - process.returncode, process.stdout.read().decode("utf-8") - ) - ) try: client.get(url, **kwargs) return - except httpx2.TransportError: + except httpx.ConnectError: time.sleep(0.1) - raise AssertionError(f"Timed out waiting for {url} to respond") - - -def find_free_port(): - with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - return sock.getsockname()[1] + raise AssertionError("Timed out waiting for {} to respond".format(url)) @pytest.fixture @@ -71,12 +54,10 @@ def bare_ds(): @pytest_asyncio.fixture(scope="session") async def ds_client(): - import secrets - from datasette.app import Datasette from datasette.database import Database - from .fixtures import CONFIG, METADATA, PLUGINS_DIR + import secrets ds = Datasette( metadata=METADATA, @@ -106,10 +87,7 @@ async def ds_client(): await db.execute_write_fn(prepare) await ds.invoke_startup() - try: - yield ds.client - finally: - ds.close() + return ds.client def pytest_report_header(config): @@ -118,8 +96,8 @@ def pytest_report_header(config): conn.close() sqlite_utils_version = importlib.metadata.version("sqlite-utils") headers = [ - f"SQLite: {version}", - f"sqlite-utils: {sqlite_utils_version}", + "SQLite: {}".format(version), + "sqlite-utils: {}".format(sqlite_utils_version), ] if config.getoption("--playwright"): try: @@ -197,8 +175,8 @@ def restore_working_directory(tmpdir, request): @pytest.fixture(scope="session", autouse=True) def check_actions_are_documented(): - from datasette.default_actions import register_actions as default_register_actions from datasette.plugins import pm + from datasette.default_actions import register_actions as default_register_actions content = ( pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" @@ -224,7 +202,7 @@ def check_actions_are_documented(): if kwargs["action"] in core_actions: assert ( action in documented_actions - ), f"Undocumented permission action: {action}" + ), "Undocumented permission action: {}".format(action) pm.add_hookcall_monitoring( before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None @@ -269,24 +247,12 @@ def ds_localhost_http_server(): # Avoid FileNotFoundError: [Errno 2] No such file or directory: cwd=tempfile.gettempdir(), ) - try: - wait_until_responds("http://localhost:8041/", process=ds_proc) - yield ds_proc - finally: - stop_process(ds_proc) - - -def stop_process(proc): - try: - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - finally: - proc.stdout.close() + wait_until_responds("http://localhost:8041/") + # Check it started successfully + assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") + yield ds_proc + # Shut it down at the end of the pytest session + ds_proc.terminate() @pytest.fixture(scope="session") @@ -307,27 +273,11 @@ def ds_unix_domain_socket_server(tmp_path_factory): cwd=tempfile.gettempdir(), ) # Poll until available - transport = httpx2.HTTPTransport(uds=uds) - client = httpx2.Client(transport=transport) + transport = httpx.HTTPTransport(uds=uds) + client = httpx.Client(transport=transport) try: - # Probe with a socket we own: the HTTP transport can leak a socket - # when connect() fails before the UDS server has started listening. - start = time.monotonic() - while True: - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe: - probe.settimeout(0.1) - probe.connect(uds) - break - except OSError: - if ds_proc.poll() is not None or time.monotonic() - start > 30: - raise - time.sleep(0.1) wait_until_responds( - "http://localhost/_memory.json", - timeout=30.0, - client=client, - process=ds_proc, + "http://localhost/_memory.json", timeout=30.0, client=client ) # Check it started successfully assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") @@ -335,75 +285,20 @@ def ds_unix_domain_socket_server(tmp_path_factory): finally: client.close() # Shut it down at the end of the pytest session - stop_process(ds_proc) + ds_proc.terminate() + try: + ds_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + ds_proc.kill() + ds_proc.wait() try: os.unlink(uds) except FileNotFoundError: pass -@pytest.fixture -def serve_with_plugins(tmp_path): - """Factory fixture for starting ``datasette serve`` in a subprocess with - plugins written to a temporary ``--plugins-dir``. - - For tests that need the real serve path: event-loop wiring, exit codes, - signals. The usual in-process ``pm.register`` plugin pattern can't reach - a subprocess, so plugin source is written out as importable files instead. - - Unlike ``ds_localhost_http_server`` this is function-scoped and takes a - fresh port each time, because each test needs its own plugins. Call it as:: - - proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE}) - - ``plugins`` maps module name to Python source. Pass - ``wait_for_startup=False`` when the server is expected to fail during - startup rather than begin serving. Extra CLI arguments are passed through. - Every process started is terminated when the test ends. - """ - processes = [] - - def start(plugins, *extra_args, wait_for_startup=True): - plugins_dir = tmp_path / "plugins" - plugins_dir.mkdir(exist_ok=True) - for module_name, source in plugins.items(): - (plugins_dir / f"{module_name}.py").write_text(source, "utf-8") - port = find_free_port() - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "datasette", - "--memory", - "--plugins-dir", - str(plugins_dir), - "-h", - "127.0.0.1", - "-p", - str(port), - *extra_args, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - # Avoid FileNotFoundError: [Errno 2] No such file or directory: - cwd=tempfile.gettempdir(), - ) - processes.append(proc) - if wait_for_startup: - wait_until_responds( - f"http://127.0.0.1:{port}/-/versions.json", process=proc - ) - return proc, port - - yield start - - for proc in processes: - stop_process(proc) - - # Import fixtures from fixtures.py to make them available -from .fixtures import ( # noqa: F401 - TEMP_PLUGIN_SECRET_FILE, +from .fixtures import ( # noqa: E402, F401 app_client, app_client_base_url_prefix, app_client_conflicting_database_names, @@ -420,4 +315,5 @@ from .fixtures import ( # noqa: F401 app_client_with_dot, app_client_with_trace, make_app_client, + TEMP_PLUGIN_SECRET_FILE, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index f8710ad4..8ab3633f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,13 +1,3 @@ -import contextlib -import json -import os -import pathlib -import tempfile -import textwrap - -import click -import pytest - from datasette.app import Datasette from datasette.fixtures import ( EXTRA_DATABASE_SQL, @@ -15,6 +5,14 @@ from datasette.fixtures import ( write_fixture_database, ) from datasette.utils.testing import TestClient +import click +import contextlib +import json +import os +import pathlib +import pytest +import tempfile +import textwrap # This temp file is used by one of the plugin config tests TEMP_PLUGIN_SECRET_FILE = os.path.join(tempfile.gettempdir(), "plugin-secret") @@ -169,10 +167,12 @@ def make_app_client( template_dir=template_dir, crossdb=crossdb, ) - try: - yield TestClient(ds) - finally: - ds.close() + yield TestClient(ds) + # Close as many database connections as possible + # to try and avoid too many open files error + for db in ds.databases.values(): + if not db.is_memory: + db.close() @pytest.fixture(scope="session") @@ -184,10 +184,9 @@ def app_client(): @pytest.fixture(scope="session") def app_client_no_files(): ds = Datasette([]) - try: - yield TestClient(ds) - finally: - ds.close() + yield TestClient(ds) + for db in ds.databases.values(): + db.close() @pytest.fixture(scope="session") diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index baf20a77..c89ad0a3 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -1,16 +1,16 @@ import asyncio +from datasette import hookimpl +from datasette.facets import Facet +from datasette.tokens import TokenHandler +from datasette import tracer +from datasette.permissions import Action +from datasette.resources import DatabaseResource +from datasette.utils import path_with_added_args +from datasette.utils.asgi import asgi_send_json, Response import base64 import json import urllib.parse -from datasette import hookimpl, tracer -from datasette.facets import Facet -from datasette.permissions import Action -from datasette.resources import DatabaseResource -from datasette.tokens import TokenHandler -from datasette.utils import path_with_added_args -from datasette.utils.asgi import Response, asgi_send_json - @hookimpl def prepare_connection(conn, database, datasette): @@ -305,7 +305,11 @@ def startup(datasette): datasette._startup_hook_fired = True # And test some import shortcuts too - from datasette import Forbidden, NotFound, Response, actor_matches_allow, hookimpl + from datasette import Response + from datasette import Forbidden + from datasette import NotFound + from datasette import hookimpl + from datasette import actor_matches_allow _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow) @@ -369,7 +373,7 @@ def table_actions(datasette, database, table, actor, request): "label": "Plugin button", "description": "Runs JavaScript from a plugin", "attrs": { - "aria-label": f"Plugin button for {table}", + "aria-label": "Plugin button for {}".format(table), "data-plugin-action": "plugin-button", "data-database": database, "data-table": table, diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 26e45a5b..864637a6 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -1,10 +1,8 @@ -import json -from functools import wraps - -import markupsafe - from datasette import hookimpl from datasette.utils.asgi import Response +from functools import wraps +import markupsafe +import json @hookimpl @@ -35,7 +33,11 @@ def render_cell(value, database): if set(data.keys()) != {"href", "label"}: return None href = data["href"] - if not (href.startswith(("/", "http://", "https://"))): + if not ( + href.startswith("/") + or href.startswith("http://") + or href.startswith("https://") + ): return None return markupsafe.Markup( '{label}'.format( @@ -52,7 +54,7 @@ def extra_template_vars(template, database, table, view_name, request, datasette datasette._last_request = request async def query_database(sql): - first_db = next(iter(datasette.databases.keys())) + first_db = list(datasette.databases.keys())[0] return (await datasette.execute(first_db, sql)).rows[0][0] async def inner(): @@ -170,10 +172,10 @@ def register_routes(datasette): path = config["path"] def new_table(request): - return Response.text(f"/db/table: {sorted(request.url_vars.items())}") + return Response.text("/db/table: {}".format(sorted(request.url_vars.items()))) return [ - (rf"/{path}/$", lambda: Response.text(path.upper())), + (r"/{}/$".format(path), lambda: Response.text(path.upper())), # Also serves to demonstrate over-ride of default paths: (r"/(?P[^/]+)/(?P[^/]+?$)", new_table), ] diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py index 671f2d1e..cfe15215 100644 --- a/tests/plugins/register_output_renderer.py +++ b/tests/plugins/register_output_renderer.py @@ -1,7 +1,6 @@ -import json - from datasette import hookimpl from datasette.utils.asgi import Response +import json async def can_render( @@ -19,7 +18,9 @@ async def can_render( "request": request, "view_name": view_name, } - return not request.args.get("_no_can_render") + if request.args.get("_no_can_render"): + return False + return True async def render_test_all_parameters( diff --git a/tests/plugins/sleep_sql_function.py b/tests/plugins/sleep_sql_function.py index 2fca1d66..d4b32a09 100644 --- a/tests/plugins/sleep_sql_function.py +++ b/tests/plugins/sleep_sql_function.py @@ -1,6 +1,5 @@ -import time - from datasette import hookimpl +import time @hookimpl diff --git a/tests/test_actions_sql.py b/tests/test_actions_sql.py index 76320bd1..a1fca971 100644 --- a/tests/test_actions_sql.py +++ b/tests/test_actions_sql.py @@ -10,11 +10,10 @@ These tests verify: import pytest import pytest_asyncio - -from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.resources import DatabaseResource, QueryResource, TableResource +from datasette import hookimpl def test_resource_string_representations(): @@ -91,7 +90,7 @@ async def test_allowed_resources_global_allow(test_ds): assert all(isinstance(t, TableResource) for t in tables) # Check specific tables are present - table_set = {(t.parent, t.child) for t in tables} + table_set = set((t.parent, t.child) for t in tables) assert ("analytics", "events") in table_set assert ("analytics", "users") in table_set assert ("analytics", "sensitive") in table_set diff --git a/tests/test_actor_restriction_bug.py b/tests/test_actor_restriction_bug.py index 6e633ff6..0bfc9e1e 100644 --- a/tests/test_actor_restriction_bug.py +++ b/tests/test_actor_restriction_bug.py @@ -6,7 +6,6 @@ config allow blocks can bypass table-level restrictions. """ import pytest - from datasette.app import Datasette from datasette.resources import TableResource diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 2f20e8e8..08b952fb 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -10,8 +10,6 @@ Layer 3: table/database views precompute all registered actions before import pytest import pytest_asyncio - -from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import ( Action, @@ -20,6 +18,7 @@ from datasette.permissions import ( _permission_check_cache, ) from datasette.resources import DatabaseResource, TableResource +from datasette import hookimpl class CountingRulesPlugin: @@ -115,7 +114,7 @@ async def test_allowed_not_memoized_without_cache(counting_ds): async def test_cache_keyed_on_full_actor_identity(counting_ds): """Interleaved checks for different actors never share cache entries.""" # Uses drop-table because default permissions deny it to non-root actors - ds, _plugin = counting_ds + ds, plugin = counting_ds resource = TableResource("analytics", "users") token = _permission_check_cache.set({}) try: @@ -181,7 +180,7 @@ async def test_cache_keyed_on_resource(counting_ds): @pytest.mark.asyncio async def test_skip_permission_checks_bypasses_cache(counting_ds): - ds, _plugin = counting_ds + ds, plugin = counting_ds resource = TableResource("analytics", "users") token = _permission_check_cache.set({}) try: diff --git a/tests/test_allowed_resources.py b/tests/test_allowed_resources.py index e251deab..e247aa78 100644 --- a/tests/test_allowed_resources.py +++ b/tests/test_allowed_resources.py @@ -7,10 +7,9 @@ based on permission rules from plugins and configuration. import pytest import pytest_asyncio - -from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import PermissionSQL +from datasette import hookimpl # Test plugin that provides permission rules diff --git a/tests/test_api.py b/tests/test_api.py index 690cb080..5ed14283 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,16 +1,13 @@ -import pathlib -import urllib - -import pytest - from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.resources import DatabaseResource, TableResource -from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode +from datasette.utils import UNSTABLE_API_MESSAGE from datasette.utils.sqlite import sqlite_version from datasette.version import __version__ - -from .fixtures import EXPECTED_PLUGINS, make_app_client +from .fixtures import make_app_client, EXPECTED_PLUGINS +import pathlib +import pytest +import sys +import urllib @pytest.mark.asyncio @@ -19,7 +16,7 @@ async def test_homepage(ds_client): assert response.status_code == 200 assert "application/json; charset=utf-8" == response.headers["content-type"] data = response.json() - assert sorted(data.get("metadata").keys()) == [ + assert sorted(list(data.get("metadata").keys())) == [ "about", "about_url", "description_html", @@ -102,11 +99,14 @@ async def test_database_page(ds_client): "tags", } - # The external-content index is visible, but its shadow tables need a - # second dependency hop and are excluded by the one-hop permission policy. + # Expected hidden tables expected_hidden_tables = { "no_primary_key", "searchable_fts", + "searchable_fts_config", + "searchable_fts_data", + "searchable_fts_docsize", + "searchable_fts_idx", } # Verify all expected tables exist @@ -384,7 +384,9 @@ async def test_row_pk_arity_mismatch_returns_400(ds_client, row_path, suffix): # because the SQL had one bind placeholder per PK column but params were # only bound for the supplied components. It should be a 400 instead, # mirroring the existing guard in datasette/views/table.py. - response = await ds_client.get(f"/fixtures/compound_primary_key/{row_path}{suffix}") + response = await ds_client.get( + "/fixtures/compound_primary_key/{}{}".format(row_path, suffix) + ) assert response.status_code == 400 if suffix == ".json": assert response.json()["ok"] is False @@ -456,67 +458,6 @@ async def test_row_foreign_key_tables(ds_client): ] -@pytest.mark.asyncio -async def test_row_foreign_key_tables_omit_denied_tables(request): - actor = {"id": "reader"} - ds = Datasette( - memory=True, - default_deny=True, - config={ - "databases": { - "data": { - "tables": { - "parents": {"permissions": {"view-table": True}}, - "private_children": {"permissions": {"view-table": False}}, - } - } - } - }, - ) - request.addfinalizer(ds.close) - db = ds.add_memory_database("fk_count_leak", name="data") - await db.execute_write("create table parents (id integer primary key, name text)") - await db.execute_write(""" - create table private_children ( - id integer primary key, - parent_id integer references parents(id) - ) - """) - await db.execute_write("insert into parents values (1, 'Public parent')") - await db.execute_write(""" - insert into private_children (id, parent_id) values - (1, 1), - (2, 1), - (3, 1) - """) - await ds.invoke_startup() - - parent = TableResource(database="data", table="parents") - private_children = TableResource(database="data", table="private_children") - assert await ds.allowed(action="view-table", resource=parent, actor=actor) - assert not await ds.allowed( - action="view-table", resource=private_children, actor=actor - ) - assert not await ds.allowed( - action="execute-sql", - resource=DatabaseResource(database="data"), - actor=actor, - ) - - direct_child = await ds.client.get("/data/private_children.json", actor=actor) - assert direct_child.status_code == 403 - parent_response = await ds.client.get( - "/data/parents/1.json?_extra=foreign_key_tables", actor=actor - ) - assert parent_response.status_code == 200 - - foreign_key_tables = parent_response.json().get("foreign_key_tables", []) - assert foreign_key_tables == [], ( - "denied child table name, foreign-key column, and row count disclosed: " - f"{foreign_key_tables}" - ) - - @pytest.mark.asyncio async def test_row_extras(ds_client): response = await ds_client.get( @@ -659,7 +600,8 @@ async def test_threads_json(ds_client): finally: ds_client.ds.root_enabled = False expected_keys = {"ok", "threads", "num_threads"} - expected_keys.update({"tasks", "num_tasks"}) + if sys.version_info >= (3, 7, 0): + expected_keys.update({"tasks", "num_tasks"}) data = response.json() assert set(data.keys()) == expected_keys # Should be at least one _execute_writes thread for __INTERNAL__ @@ -672,13 +614,13 @@ async def test_plugins_json(ds_client): response = await ds_client.get("/-/plugins.json") # Filter out TrackEventPlugin actual_plugins = sorted( - [p for p in response.json() if p["name"] != "TrackEventPlugin"], + [p for p in response.json()["plugins"] if p["name"] != "TrackEventPlugin"], key=lambda p: p["name"], ) assert EXPECTED_PLUGINS == actual_plugins # Try with ?all=1 response = await ds_client.get("/-/plugins.json?all=1") - names = {p["name"] for p in response.json()} + names = {p["name"] for p in response.json()["plugins"]} assert names.issuperset(p["name"] for p in EXPECTED_PLUGINS) assert names.issuperset(DEFAULT_PLUGINS) @@ -953,7 +895,10 @@ async def test_hidden_sqlite_stat1_table(): await db.execute_write("analyze") data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] - assert tables == [("normal", False)] + assert tables in ( + [("normal", False), ("sqlite_stat1", True)], + [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)], + ) @pytest.mark.asyncio @@ -985,33 +930,6 @@ async def test_tilde_encoded_database_names(db_name): assert response2.status_code == 200 -@pytest.mark.asyncio -@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar")) -async def test_table_with_reserved_characters_in_name(table_name): - # Table names containing characters such as "]" that cannot be escaped - # using SQLite [bracket] quoting used to break schema introspection and - # the table page - https://github.com/simonw/datasette/issues/2431 - ds = Datasette() - db = ds.add_memory_database("test_reserved_table_names") - await db.execute_write( - f"create table {escape_sqlite(table_name)} (id integer primary key, name text)" - ) - await db.execute_write( - f"insert into {escape_sqlite(table_name)} (id, name) values (1, 'one')" - ) - # Schema introspection (populate_schema_tables) must not crash: - db_response = await ds.client.get("/test_reserved_table_names.json") - assert db_response.status_code == 200 - tables = {t["name"]: t for t in db_response.json()["tables"]} - assert tables[table_name]["count"] == 1 - # And the table page itself must load and return the row: - table_response = await ds.client.get( - f"/test_reserved_table_names/{tilde_encode(table_name)}.json?_shape=array" - ) - assert table_response.status_code == 200 - assert table_response.json() == [{"id": 1, "name": "one"}] - - @pytest.mark.asyncio @pytest.mark.parametrize( "config,expected", diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 50bf1d67..a803fbbc 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,25 +1,21 @@ -import time - -import pytest -import sqlite_utils - from datasette.app import Datasette from datasette.events import RenameTableEvent from datasette.utils import error_body, escape_sqlite, sqlite3 - from .utils import last_event +import pytest +import time def assert_schema_contains(fragment, schema): - assert ( - fragment in schema - ), f"Expected schema to contain {fragment!r}, got {schema!r}" + assert fragment in schema, "Expected schema to contain {!r}, got {!r}".format( + fragment, schema + ) def assert_schema_not_contains(fragment, schema): assert ( fragment not in schema - ), f"Expected schema not to contain {fragment!r}, got {schema!r}" + ), "Expected schema not to contain {!r}, got {!r}".format(fragment, schema) @pytest.fixture @@ -51,114 +47,17 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", } -@pytest.mark.asyncio -@pytest.mark.parametrize("operation", ["read", "read_row", "rename"]) -async def test_trailing_lf_table_permissions(tmp_path, operation): - # SQLite treats "secret" and "secret\n" as different table names. Permission - # checks and SQL execution must agree on which table a request targets. - db_path = tmp_path / "data.db" - conn = sqlite3.connect(str(db_path)) - conn.executescript( - "create table secret (id integer primary key, value text);" - "insert into secret values (1, 'private');" - ) - conn.close() - # Allow builder to create and use tables generally, but explicitly deny - # access to the existing secret table below. Disable arbitrary SQL access. - grants = { - action: {"id": "builder"} - for action in ( - "view-database", - "create-table", - "view-table", - "insert-row", - "alter-table", - ) - } - ds = Datasette( - [str(db_path)], - default_deny=True, - settings={"default_allow_sql": False}, - config={ - "permissions": {"view-instance": {"id": "builder"}}, - "databases": { - "data": { - "permissions": grants, - "tables": { - "secret": { - "permissions": { - "view-table": False, - "insert-row": False, - "alter-table": False, - } - } - }, - } - }, - }, - ) - headers = _headers(write_token(ds, actor_id="builder")) - try: - # Establish that the protected table is inaccessible before creating - # a second table whose name differs only by a trailing line feed. - response = await ds.client.get("/data/secret.json", headers=headers) - assert response.status_code == 403 - response = await ds.client.get( - "/data/-/query.json?sql=select+*+from+secret", headers=headers - ) - assert response.status_code == 403 - # Distinct values let us detect if an operation targets secret - # instead of the newly created secret\n table. - response = await ds.client.post( - "/data/-/create", - json={"table": "secret\n", "row": {"id": 1, "value": "decoy"}, "pk": "id"}, - headers=headers, - ) - assert response.status_code == 201, response.text - # ~0A is Datasette's URL encoding for the line feed in the table name. - if operation in ("read", "read_row"): - # Both table and row endpoints must return only the permitted row. - path = "/1.json" if operation == "read_row" else ".json" - response = await ds.client.get( - "/data/secret~0A" + path + "?_shape=array", headers=headers - ) - assert response.status_code == 200, response.text - assert response.json() == [{"id": 1, "value": "decoy"}] - else: - # Renaming must move the permitted table, preserving its contents - # and removing its old name from the database. - response = await ds.client.post( - "/data/secret~0A/-/alter", - json={"operations": [{"op": "rename_table", "args": {"to": "moved"}}]}, - headers=headers, - ) - assert response.status_code == 200, response.text - db = ds.get_database("data") - assert ( - await db.execute('select value from "moved"') - ).single_value() == "decoy" - assert "secret\n" not in await db.table_names() - # Verify that the protected table and its data are unchanged, and that - # the API still denies access to it. - db = ds.get_database("data") - assert ( - await db.execute('select value from "secret"') - ).single_value() == "private" - response = await ds.client.get("/data/secret.json", headers=headers) - assert response.status_code == 403 - finally: - ds.close() - - def _insert_and_fetch_created(conn, table, insert_sql): cursor = conn.execute(insert_sql) return conn.execute( - f"select created, typeof(created) from {escape_sqlite(table)} where rowid = ?", + "select created, typeof(created) from {} where rowid = ?".format( + escape_sqlite(table) + ), (cursor.lastrowid,), ).fetchone() @@ -167,82 +66,6 @@ BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"} BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}' -@pytest.mark.asyncio -@pytest.mark.parametrize("use_fallback", (False, True)) -@pytest.mark.parametrize( - "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase") -) -@pytest.mark.parametrize( - "module,definition,values,shadow_suffix", - ( - ("fts5", "body", "'original'", "_content"), - ("fts4", "body", "'original'", "_content"), - ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"), - ), -) -@pytest.mark.parametrize("shadow", (False, True)) -async def test_structured_writes_require_ordinary_tables( - ds_write, - monkeypatch, - use_fallback, - operation, - module, - definition, - values, - shadow_suffix, - shadow, -): - if use_fallback: - monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False) - db = ds_write.get_database("data") - await db.execute_write(f"create virtual table indexed using {module}({definition})") - await db.execute_write(f"insert into indexed values ({values})") - table = "indexed" + (shadow_suffix if shadow else "") - row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0] - pks = await db.primary_keys(table) - pk_value = row[pks[0] if pks else "rowid"] - before = await db.execute_fn(lambda conn: list(conn.iterdump())) - - if operation in ("create", "create_uppercase"): - path = "/data/-/create" - body = { - "table": table.upper() if operation == "create_uppercase" else table, - "rows": [row], - } - elif operation in ("update", "delete"): - path = f"/data/{table}/{pk_value}/-/{operation}" - body = {"update": row} if operation == "update" else {} - else: - path = f"/data/{table}/-/{operation}" - body = {"rows": [row]} - response = await ds_write.client.post( - path, json=body, headers=_headers(write_token(ds_write)) - ) - assert response.status_code == 400, response.text - assert response.json()["errors"] == ["Structured writes require an ordinary table"] - assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before - - -@pytest.mark.asyncio -async def test_structured_writes_to_content_table_maintain_fts(ds_write): - db = ds_write.get_database("data") - await db.execute_write_fn( - lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts( - ["title"], create_triggers=True - ) - ) - response = await ds_write.client.post( - "/data/docs/-/insert", - json={"row": {"id": 1, "title": "ordinary content"}}, - headers=_headers(write_token(ds_write)), - ) - assert response.status_code == 201, response.text - matches = await db.execute( - "select rowid from docs_fts where docs_fts match ?", ["ordinary"] - ) - assert [row[0] for row in matches.rows] == [1] - - @pytest.mark.asyncio async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write): token = write_token(ds_write) @@ -418,7 +241,7 @@ async def test_insert_row(ds_write, content_type): "/data/docs/-/insert", json={"row": {"title": "Test", "score": 1.2, "age": 5}}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": content_type, }, ) @@ -463,7 +286,11 @@ async def test_insert_row_alter(ds_write): @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): token = write_token(ds_write) - data = {"rows": [{"title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20)]} + data = { + "rows": [ + {"title": "Test {}".format(i), "score": 1.0, "age": 5} for i in range(20) + ] + } if return_rows: data["return"] = True response = await ds_write.client.post( @@ -487,7 +314,8 @@ async def test_insert_rows(ds_write, return_rows): ).dicts() assert len(actual_rows) == 20 assert actual_rows == [ - {"id": i + 1, "title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20) + {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} + for i in range(20) ] assert response.json()["ok"] is True if return_rows: @@ -733,13 +561,13 @@ async def test_insert_or_upsert_row_errors( ) if special_case == "bad_token": token += "bad" - kwargs = { - "json": input, - "headers": { - "Authorization": f"Bearer {token}", + kwargs = dict( + json=input, + headers={ + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", }, - } + ) if special_case != "bad_token": actor_response = ( @@ -794,7 +622,7 @@ async def test_upsert_permissions_per_table(ds_write, allowed): "/data/docs/-/upsert", json={"rows": [{"id": 1, "title": "One"}]}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), }, ) if allowed: @@ -1031,7 +859,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Should be a single row assert ( await ds_write.client.get( - f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}" + "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( + table + ) ) ).json() == [1] # Now delete the row @@ -1039,12 +869,14 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Special case for that rowid table delete_path = ( await ds_write.client.get( - f"/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{table}" + "/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}".format( + table + ) ) ).json()[0] delete_response = await ds_write.client.post( - f"/data/{table}/{delete_path}/-/delete", + "/data/{}/{}/-/delete".format(table, delete_path), headers=_headers(write_token(ds_write)), ) assert delete_response.status_code == 200 @@ -1057,7 +889,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( - f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}" + "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( + table + ) ) ).json() == [0] @@ -1107,7 +941,7 @@ async def test_update_row_invalid_key(ds_write): pk = await _insert_row(ds_write) - path = f"/data/docs/{pk}/-/update" + path = "/data/docs/{}/-/update".format(pk) response = await ds_write.client.post( path, json={"update": {"title": "New title"}, "bad_key": 1}, @@ -1126,7 +960,7 @@ async def test_update_row_invalid_key(ds_write): async def test_update_row_alter(ds_write): token = write_token(ds_write, permissions=["ur", "at"]) pk = await _insert_row(ds_write) - path = f"/data/docs/{pk}/-/update" + path = "/data/docs/{}/-/update".format(pk) response = await ds_write.client.post( path, json={"update": {"title": "New title", "extra": "extra"}, "alter": True}, @@ -1282,9 +1116,9 @@ async def test_alter_table_integer_default_expr( assert expected_schema in data["schema"] columns = await db.execute("select * from pragma_table_info('docs')") - created_column = next( + created_column = [ column for column in columns.dicts() if column["name"] == "created" - ) + ][0] assert created_column["type"] == "INTEGER" assert expected_schema in created_column["dflt_value"] @@ -1471,7 +1305,7 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w @pytest.mark.asyncio async def test_foreign_key_suggestions(ds_write): - token = write_token(ds_write, permissions=["alter-table", "view-table"]) + token = write_token(ds_write, permissions=["at"]) db = ds_write.get_database("data") await db.execute_write("create table owners (id integer primary key)") await db.execute_write("insert into owners (id) values (1), (2), (3)") @@ -1537,7 +1371,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write): @pytest.mark.asyncio async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): - token = write_token(ds_write, permissions=["alter-table", "view-table"]) + token = write_token(ds_write, permissions=["at"]) db = ds_write.get_database("data") await db.execute_write("create table owners (id integer primary key)") @@ -1568,7 +1402,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): @pytest.mark.asyncio async def test_foreign_key_targets(ds_write): - token = write_token(ds_write, permissions=["create-table", "view-table"]) + token = write_token(ds_write, permissions=["ct"]) db = ds_write.get_database("data") await db.execute_write("create table owners (id integer primary key)") await db.execute_write("create table categories (slug varchar(30) primary key)") @@ -1585,8 +1419,7 @@ async def test_foreign_key_targets(ds_write): await db.execute_write("create table no_pk (name text)") try: await db.execute_write("create virtual table search_docs using fts5(body)") - except Exception: # noqa: BLE001, S110 - # FTS5 is not available in every SQLite build + except Exception: pass response = await ds_write.client.get( @@ -1792,7 +1625,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return): token = write_token(ds_write) pk = await _insert_row(ds_write) - path = f"/data/docs/{pk}/-/update" + path = "/data/docs/{}/-/update".format(pk) data = {"update": input} if use_return: @@ -1827,7 +1660,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return): # And fetch the row to check it's updated response = await ds_write.client.get( - f"/data/docs/{pk}.json?_shape=array", + "/data/docs/{}.json?_shape=array".format(pk), ) assert response.status_code == 200 row = response.json()[0] @@ -1901,42 +1734,6 @@ async def test_drop_table(ds_write, scenario): assert (await ds_write.client.get("/data/docs")).status_code == 404 -@pytest.mark.asyncio -async def test_drop_table_cleans_up_fts(ds_write): - db = ds_write.get_database("data") - - def enable_fts(conn): - sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True) - - await db.execute_write_fn(enable_fts) - assert { - row[0] - for row in await db.execute( - "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" - ) - } == { - "docs_fts", - "docs_fts_config", - "docs_fts_data", - "docs_fts_docsize", - "docs_fts_idx", - } - - response = await ds_write.client.post( - "/data/docs/-/drop", - json={"confirm": True}, - headers=_headers(write_token(ds_write)), - ) - - assert response.json() == {"ok": True} - assert [ - row[0] - for row in await db.execute( - "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" - ) - ] == [] - - @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_status,expected_response,expected_events", @@ -2509,7 +2306,7 @@ async def test_create_table_integer_default_expr( ds_write, default_expr, minimum_value, expected_schema ): token = write_token(ds_write) - table = f"default_{default_expr}" + table = "default_{}".format(default_expr) response = await ds_write.client.post( "/data/-/create", json={ @@ -2537,7 +2334,7 @@ async def test_create_table_integer_default_expr( row = await db.execute_write_fn( lambda conn: _insert_and_fetch_created( - conn, table, f"insert into {escape_sqlite(table)} default values" + conn, table, "insert into {} default values".format(escape_sqlite(table)) ) ) assert row[0] > minimum_value @@ -2920,119 +2717,3 @@ async def test_create_using_alter_against_existing_table( insert_rows_event = ds_write._tracked_events[1] assert insert_rows_event.name == "insert-rows" assert insert_rows_event.num_rows == 1 - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("denied_action", "request_body"), - ( - ( - "insert-row", - { - "table": "salaries", - "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}], - }, - ), - ( - "update-row", - { - "table": "salaries", - "rows": [{"id": 1, "note": "REPLACED"}], - "pk": "id", - "replace": True, - }, - ), - ( - "alter-table", - { - "table": "salaries", - "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}], - "alter": True, - }, - ), - ), -) -async def test_create_table_existing_table_respects_table_level_denial( - denied_action, request_body -): - # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table - # inserts rows into it, so insert-row (and update-row / alter-table) must be - # checked against the TableResource, not just the DatabaseResource. - ds = Datasette( - memory=True, - config={ - "databases": { - # id=editor user has each permission at the database level, but - # the selected action is explicitly denied on the salaries table - "data": { - "permissions": { - "create-table": {"id": "editor"}, - "insert-row": {"id": "editor"}, - "update-row": {"id": "editor"}, - "alter-table": {"id": "editor"}, - }, - "tables": { - "salaries": {"permissions": {denied_action: False}}, - }, - } - } - }, - ) - db = ds.add_memory_database( - f"create_table_existing_table_denied_{denied_action}", name="data" - ) - await db.execute_write("create table salaries (id integer primary key, note text)") - await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')") - await ds.invoke_startup() - - if denied_action == "insert-row": - # Sanity: direct insert into salaries is denied for this actor - direct = await ds.client.post( - "/data/salaries/-/insert", - actor={"id": "editor"}, - json={"row": {"id": 9, "note": "INJ-DIRECT"}}, - ) - assert direct.status_code == 403 - - response = await ds.client.post( - "/data/-/create", - actor={"id": "editor"}, - json=request_body, - ) - assert response.status_code == 403, response.json() - assert response.json()["errors"] == [f"Permission denied: need {denied_action}"] - rows = (await db.execute("select id, note from salaries order by id")).rows - assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")] - assert await db.table_columns("salaries") == ["id", "note"] - - -@pytest.mark.asyncio -async def test_create_table_respects_predeclared_table_level_denial(): - ds = Datasette( - memory=True, - config={ - "databases": { - "data": { - "permissions": { - "create-table": {"id": "editor"}, - "insert-row": {"id": "editor"}, - }, - "tables": { - "planned_table": {"permissions": {"insert-row": False}}, - }, - } - } - }, - ) - db = ds.add_memory_database("create_table_predeclared_denial", name="data") - await ds.invoke_startup() - - response = await ds.client.post( - "/data/-/create", - actor={"id": "editor"}, - json={"table": "planned_table", "rows": [{"id": 1}]}, - ) - - assert response.status_code == 403, response.json() - assert response.json()["errors"] == ["Permission denied: need insert-row"] - assert not await db.table_exists("planned_table") diff --git a/tests/test_auth.py b/tests/test_auth.py index 68a2e6fd..d2913ecc 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,18 +1,14 @@ -import time -from unittest.mock import AsyncMock - -import pytest from bs4 import BeautifulSoup as Soup +from .utils import cookie_was_deleted, last_event from click.testing import CliRunner - +from datasette.utils import baseconv from datasette.cli import cli from datasette.resources import ( DatabaseResource, TableResource, ) -from datasette.utils import baseconv - -from .utils import cookie_was_deleted, last_event +import pytest +import time @pytest.mark.asyncio @@ -208,7 +204,7 @@ def test_auth_create_token( assert response2.status == 200 if errors: for error in errors: - assert f'

      {error}

      ' in response2.text + assert '

      {}

      '.format(error) in response2.text else: # Check create-token event event = last_event(app_client.ds) @@ -232,41 +228,12 @@ def test_auth_create_token( # And test that token response3 = app_client.get( "/-/actor.json", - headers={"Authorization": "Bearer {}".format(f"dstok_{token}")}, + headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, ) assert response3.status == 200 assert response3.json["actor"]["id"] == "test" -@pytest.mark.asyncio -@pytest.mark.parametrize("method", ["GET", "POST"]) -@pytest.mark.parametrize( - "restrictions", - [ - {}, - {"a": ["vi"]}, - {"d": {"db": ["vd"]}}, - {"r": {"db": {"t1": ["vt"]}}}, - ], - ids=["empty", "instance", "database", "table"], -) -async def test_auth_create_token_not_allowed_for_restricted_actors( - bare_ds, monkeypatch, method, restrictions -): - create_token = AsyncMock() - monkeypatch.setattr(bare_ds, "create_token", create_token) - - response = await bare_ds.client.request( - method, - "/-/create-token", - actor={"id": "test", "_r": restrictions}, - ) - - assert response.status_code == 403 - assert "Restricted actors cannot create API tokens" in response.text - create_token.assert_not_called() - - @pytest.mark.asyncio async def test_auth_create_token_not_allowed_for_tokens(ds_client): ds_tok = ds_client.ds.sign( @@ -274,7 +241,7 @@ async def test_auth_create_token_not_allowed_for_tokens(ds_client): ) response = await ds_client.get( "/-/create-token", - headers={"Authorization": f"Bearer dstok_{ds_tok}"}, + headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, ) assert response.status_code == 403 @@ -319,12 +286,12 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): elif scenario == "invalid_token": token = "invalid" if token: - token = f"dstok_{token}" + token = "dstok_{}".format(token) if scenario == "allow_signed_tokens_off": ds_client.ds._settings["allow_signed_tokens"] = False headers = {} if token: - headers["Authorization"] = f"Bearer {token}" + headers["Authorization"] = "Bearer {}".format(token) response = await ds_client.get("/-/actor.json", headers=headers) try: if should_work: @@ -371,7 +338,7 @@ def test_cli_create_token(app_client, expires): assert details.keys() == expected_keys assert details["a"] == "test" response = app_client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) if expires is None or expires > 0: expected_actor = { @@ -554,25 +521,3 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): ) is not True ), "Root without root_enabled should not automatically get set-column-type" - - -@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60)) -def test_set_actor_cookie_honours_expire_after(expire_after): - # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of - # seconds, but every value was being replaced with 24 hours. - from datasette.app import Datasette - from datasette.utils.asgi import Response - - ds = Datasette(memory=True) - response = Response.text("") - before = int(time.time()) - ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after) - after = int(time.time()) - - (header,) = response._set_cookie_headers - assert header.startswith("ds_actor=") - value = header[len("ds_actor=") :].split(";", 1)[0] - data = ds.unsign(value, "actor") - assert data["a"] == {"id": "test"} - expires_at = baseconv.base62.decode(data["e"]) - assert before + expire_after <= expires_at <= after + expire_after diff --git a/tests/test_background_tasks.py b/tests/test_background_tasks.py deleted file mode 100644 index c0d7ce49..00000000 --- a/tests/test_background_tasks.py +++ /dev/null @@ -1,381 +0,0 @@ -""" -Tests for datasette.add_background_task() / start_background_tasks() and the -BackgroundTask / BackgroundTaskSupervisor machinery in -datasette/background_tasks.py. -""" - -import asyncio -import contextlib -import logging - -import httpx2 -import pytest - -from datasette import hookimpl -from datasette.app import Datasette -from datasette.plugins import pm - - -async def _drive_lifespan_startup(app): - """Send a single lifespan.startup message into app's ASGI lifespan loop - and return the list of messages sent back, without ever sending - lifespan.shutdown. Copied from tests/test_lifespan.py's helper of the - same name - mirrors what a real server does: after startup completes - it parks waiting for the next event, and we cancel that wait once - we've observed the startup response. - """ - messages_sent = [] - startup_responded = asyncio.Event() - delivered = False - - async def receive(): - nonlocal delivered - if not delivered: - delivered = True - return {"type": "lifespan.startup"} - await asyncio.Event().wait() - - async def send(message): - messages_sent.append(message) - startup_responded.set() - - task = asyncio.create_task(app({"type": "lifespan"}, receive, send)) - try: - await asyncio.wait_for(startup_responded.wait(), timeout=5) - finally: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - return messages_sent - - -@pytest.mark.asyncio -async def test_tasks_registered_in_startup_hook_run_after_lifespan_startup(): - # Two tasks registered by one plugin's startup hook - order preserved, - # both running after lifespan startup completes, and no HTTP request - # of any kind is issued anywhere in this test. - events = [] - - async def task_one(datasette): - events.append("task_one") - # Wait indefinitely to simulate long-lived background work, keeping - # the task "running" for the assertions below until cleanup cancels it. - await asyncio.Event().wait() - - async def task_two(datasette): - events.append("task_two") - await asyncio.Event().wait() - - class TwoTaskPlugin: - __name__ = "TwoTaskPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - datasette.add_background_task(task_one, name="task-one") - datasette.add_background_task(task_two, name="task-two") - - return inner - - ds = Datasette(memory=True) - pm.register(TwoTaskPlugin(), name="two_task_plugin") - try: - app = ds.app() - messages = await _drive_lifespan_startup(app) - assert {"type": "lifespan.startup.complete"} in messages - - handles = ds._background_tasks.tasks() - assert [h.name for h in handles] == ["task-one", "task-two"] - - # Let both tasks run their first line of code. - await asyncio.sleep(0) - assert handles[0].state == "running" - assert handles[1].state == "running" - assert events == ["task_one", "task_two"] - finally: - pm.unregister(name="two_task_plugin") - await ds._background_tasks.cancel_all(grace=1.0) - - -@pytest.mark.asyncio -async def test_launch_waits_for_every_startup_hook_before_running_any_task(): - # PluginA registers a task from its startup hook; PluginB does the - # same from ITS startup hook, which runs after PluginA's (forced with - # tryfirst=True on A). Even though A's registration happens first, - # A's task body must not actually execute until every startup hook - - # including B's - has finished, since launch only happens after - # invoke_startup() completes. This is the ordering guarantee that - # dissolves datasette-cron's tryfirst=True launch hack. - hook_call_order = [] - seen_names_when_a_ran = {} - - async def task_a(datasette): - seen_names_when_a_ran["names"] = [ - h.name for h in datasette._background_tasks.tasks() - ] - - async def task_b(datasette): - pass - - class PluginA: - __name__ = "PluginA" - - @hookimpl(tryfirst=True) - def startup(self, datasette): - async def inner(): - hook_call_order.append("A") - datasette.add_background_task(task_a, name="task-a") - - return inner - - class PluginB: - __name__ = "PluginB" - - @hookimpl - def startup(self, datasette): - async def inner(): - hook_call_order.append("B") - datasette.add_background_task(task_b, name="task-b") - - return inner - - ds = Datasette(memory=True) - pm.register(PluginA(), name="plugin_a") - pm.register(PluginB(), name="plugin_b") - try: - await ds.start_background_tasks() - # Confirm A's startup hook really did run (and register task-a) - # strictly before B's startup hook ran. - assert hook_call_order == ["A", "B"] - - handles = ds._background_tasks.tasks() - await asyncio.wait_for(asyncio.gather(*[h.task for h in handles]), timeout=5) - # Yet by the time task-a's own body executed (after launch, which - # only happens once every startup hook - including B's - has - # finished), task-b was already registered. - assert "task-b" in seen_names_when_a_ran["names"] - finally: - pm.unregister(name="plugin_a") - pm.unregister(name="plugin_b") - - -@pytest.mark.asyncio -async def test_concurrent_first_requests_launch_background_tasks_exactly_once(): - launch_count = {"n": 0} - - async def counting_task(datasette): - launch_count["n"] += 1 - - class CountingTaskPlugin: - __name__ = "CountingTaskPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - datasette.add_background_task(counting_task, name="counting-task") - - return inner - - ds = Datasette(memory=True) - pm.register(CountingTaskPlugin(), name="counting_task_plugin") - try: - app = ds.app() - transport = httpx2.ASGITransport(app=app) - async with httpx2.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - responses = await asyncio.gather( - *[client.get("/-/versions.json") for _ in range(10)] - ) - assert all(response.status_code == 200 for response in responses) - - handles = ds._background_tasks.tasks() - assert len(handles) == 1 - await asyncio.wait_for(handles[0].task, timeout=5) - assert launch_count["n"] == 1 - finally: - pm.unregister(name="counting_task_plugin") - - -@pytest.mark.asyncio -async def test_post_launch_registration_starts_immediately_and_cancel_works(): - ds = Datasette(memory=True) - await ds.start_background_tasks() # nothing registered yet, but launched - - started = asyncio.Event() - - async def long_running(datasette): - started.set() - await asyncio.Event().wait() - - handle = ds.add_background_task(long_running, name="dynamic-task") - # Registered after launch: starts immediately rather than sitting in - # "registered" limbo. - assert handle.state == "running" - assert handle.task is not None - - await asyncio.wait_for(started.wait(), timeout=5) - assert handle.state == "running" - - handle.cancel() - with pytest.raises(asyncio.CancelledError): - await handle.task - await asyncio.sleep(0) - assert handle.state == "cancelled" - - -@pytest.mark.asyncio -async def test_pre_launch_registration_starts_as_registered(): - ds = Datasette(memory=True) - - async def task(datasette): - pass - - handle = ds.add_background_task(task, name="buffered-task") - assert handle.state == "registered" - assert handle.task is None - - handle.cancel() # not yet launched: deregisters instead of cancelling - assert handle not in ds._background_tasks.tasks() - - -@pytest.mark.asyncio -async def test_crashing_task_logs_traceback_and_state_is_crashed(caplog): - ds = Datasette(memory=True) - await ds.start_background_tasks() - - survivor_ran = asyncio.Event() - - async def crashing_task(datasette): - raise RuntimeError("kaboom") - - async def survivor(datasette): - survivor_ran.set() - - with caplog.at_level(logging.ERROR, logger="datasette.background_tasks"): - crash_handle = ds.add_background_task(crashing_task, name="crashing_task") - survivor_handle = ds.add_background_task(survivor, name="survivor") - await asyncio.wait_for( - asyncio.gather( - crash_handle.task, survivor_handle.task, return_exceptions=True - ), - timeout=5, - ) - - assert crash_handle.state == "crashed" - assert isinstance(crash_handle.exception, RuntimeError) - assert str(crash_handle.exception) == "kaboom" - - # The crash must not affect any other task. - assert survivor_ran.is_set() - assert survivor_handle.state == "completed" - - assert "crashing_task" in caplog.text - assert "kaboom" in caplog.text - assert "Traceback" in caplog.text - assert "RuntimeError" in caplog.text - - -def test_name_collisions_get_suffixed_and_explicit_names_are_respected(): - ds = Datasette(memory=True) - - async def noop(datasette): - pass - - async def another_noop(datasette): - pass - - h1 = ds.add_background_task(noop, name="dup") - h2 = ds.add_background_task(another_noop, name="dup") - h3 = ds.add_background_task(noop, name="dup") - assert [h1.name, h2.name, h3.name] == ["dup", "dup-2", "dup-3"] - - h_explicit = ds.add_background_task(noop, name="explicit-name") - assert h_explicit.name == "explicit-name" - - h_default = ds.add_background_task(noop) - assert h_default.name == noop.__qualname__ - - -@pytest.mark.asyncio -async def test_start_background_tasks_on_bare_datasette(): - # The headless-CLI path (datasette-rss's `fetch --due` shape): no - # server, no lifespan, no first HTTP request - just an explicit call. - ran = asyncio.Event() - - async def task(datasette): - ran.set() - - ds = Datasette([]) - assert ds._startup_invoked is False - - handle = ds.add_background_task(task, name="headless-task") - assert handle.state == "registered" - - await ds.start_background_tasks() - - assert ds._startup_invoked is True - await asyncio.wait_for(ran.wait(), timeout=5) - await asyncio.wait_for(handle.task, timeout=5) - # handle.task being done only guarantees the coroutine has returned, - # not that our done-callback (which updates handle.state) has run yet - - # asyncio schedules done-callbacks via call_soon, and awaiting an - # already-done future/task returns immediately without giving the loop - # a chance to drain its ready queue. Yield once to let it run. - await asyncio.sleep(0) - assert handle.state == "completed" - - -@pytest.mark.asyncio -async def test_cancel_all_cancels_running_tasks_and_leaves_completed_alone(): - ds = Datasette(memory=True) - await ds.start_background_tasks() - - async def forever(datasette): - await asyncio.Event().wait() - - async def quick(datasette): - return "done" - - forever_handle = ds.add_background_task(forever, name="forever") - quick_handle = ds.add_background_task(quick, name="quick") - await asyncio.wait_for(quick_handle.task, timeout=5) - assert quick_handle.state == "completed" - - await ds._background_tasks.cancel_all(grace=1.0) - - assert forever_handle.state == "cancelled" - assert quick_handle.state == "completed" - - -@pytest.mark.asyncio -async def test_cancel_all_logs_stragglers_that_outlive_the_grace_period(caplog): - ds = Datasette(memory=True) - await ds.start_background_tasks() - - async def stubborn(datasette): - with contextlib.suppress(asyncio.CancelledError): - await asyncio.sleep(10) - # Swallowing CancelledError above and returning normally simulates - # a task that ignores cancellation for longer than the grace period. - await asyncio.sleep(10) - - handle = ds.add_background_task(stubborn, name="stubborn-task") - # Let the task actually start running and reach its first sleep (inside - # the CancelledError-suppressing block) before cancelling it - a task - # cancelled before it has ever run its first step never enters that - # block at all (the throw happens before the coroutine body starts), - # so it would finish cancelling immediately instead of behaving like a - # straggler. - await asyncio.sleep(0) - - with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"): - await ds._background_tasks.cancel_all(grace=0.1) - - assert "stubborn-task" in caplog.text - - # Clean up: actually cancel it now that the test has made its - # assertion, so it doesn't leak past the end of the test. - handle.task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await handle.task diff --git a/tests/test_base_view.py b/tests/test_base_view.py index b46f7ce1..c1b0cf20 100644 --- a/tests/test_base_view.py +++ b/tests/test_base_view.py @@ -1,10 +1,8 @@ -import json - -import pytest - +from datasette.views.base import View from datasette import Request, Response from datasette.app import Datasette -from datasette.views.base import View +import json +import pytest class GetView(View): diff --git a/tests/test_cli.py b/tests/test_cli.py index fbd4a8a9..cbd8edad 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,28 +1,23 @@ +from .fixtures import ( + make_app_client, + TestClient as _TestClient, + EXPECTED_PLUGINS, +) +from datasette.app import SETTINGS +from datasette.plugins import DEFAULT_PLUGINS, pm +from datasette.cli import cli, serve +from datasette.version import __version__ +from datasette.utils import tilde_encode +from datasette.utils.sqlite import sqlite3 +from click.testing import CliRunner import io import json import pathlib +import pytest import sys import textwrap from unittest import mock -import pytest -from click.testing import CliRunner - -from datasette.app import SETTINGS -from datasette.cli import cli, serve -from datasette.plugins import DEFAULT_PLUGINS, pm -from datasette.utils import tilde_encode -from datasette.utils.sqlite import sqlite3 -from datasette.version import __version__ - -from .fixtures import ( - EXPECTED_PLUGINS, - make_app_client, -) -from .fixtures import ( - TestClient as _TestClient, -) - def test_inspect_cli(app_client): runner = CliRunner() @@ -465,7 +460,7 @@ def test_serve_create(tmpdir): @pytest.mark.parametrize("argument", ("-c", "--config")) @pytest.mark.parametrize("format_", ("json", "yaml")) def test_serve_config(tmpdir, argument, format_): - config_path = tmpdir / f"datasette.{format_}" + config_path = tmpdir / "datasette.{}".format(format_) config_path.write_text( ( "settings:\n default_page_size: 5\n" @@ -518,13 +513,13 @@ def test_weird_database_names(tmpdir, filename): result1 = runner.invoke(cli, [db_path, "--get", "/"]) assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] - expected_link = ( - f'{filename_no_stem}' + expected_link = '{}'.format( + tilde_encode(filename_no_stem), filename_no_stem ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", f"/{tilde_encode(filename_no_stem)}"] + cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))] ) assert result2.exit_code == 0, result2.output diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 562bd7aa..fe9416d6 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,10 +1,8 @@ -import json -import textwrap - -from click.testing import CliRunner - from datasette.cli import cli from datasette.plugins import pm +from click.testing import CliRunner +import textwrap +import json def test_serve_with_get(tmp_path_factory): @@ -46,59 +44,9 @@ def test_serve_with_get(tmp_path_factory): # Annoyingly that new test plugin stays resident - we need # to manually unregister it to avoid conflict with other tests - to_unregister = next( + to_unregister = [ p for p in pm.get_plugins() if p.__name__ == "init_for_serve_with_get.py" - ) - pm.unregister(to_unregister) - - -def test_serve_with_get_does_not_launch_background_tasks(tmp_path_factory): - # --get must never launch background tasks, even though its TestClient - # request - # flows through the full ASGI stack (including the AsgiRunOnFirstRequest - # fallback that would otherwise launch them). The plugin's startup hook - # itself still runs (registration happens) - only the launch is - # suppressed, so the sentinel file the background task would write must - # never appear. - plugins_dir = tmp_path_factory.mktemp("plugins_for_get_background_tasks") - sentinel = plugins_dir / "sentinel.txt" - (plugins_dir / "bg_task_for_get.py").write_text( - textwrap.dedent( - f""" - from datasette import hookimpl - - @hookimpl - def startup(datasette): - async def inner(): - async def task(datasette): - with open("{sentinel!s}", "w") as fp: - fp.write("ran") - - datasette.add_background_task(task, name="get-sentinel-task") - - return inner - """, - ), - "utf-8", - ) - runner = CliRunner() - result = runner.invoke( - cli, - [ - "serve", - "--memory", - "--plugins-dir", - str(plugins_dir), - "--get", - "/_memory/-/query.json?sql=select+1", - ], - ) - assert result.exit_code == 0, result.output - assert not sentinel.exists() - - to_unregister = next( - p for p in pm.get_plugins() if p.__name__ == "bg_task_for_get.py" - ) + ][0] pm.unregister(to_unregister) diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 163d3fcc..47f23c08 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,15 +1,11 @@ -import signal -import socket -import subprocess -import time - -import httpx2 +import httpx import pytest +import socket @pytest.mark.serial def test_serve_localhost_http(ds_localhost_http_server): - response = httpx2.get("http://localhost:8041/_memory.json") + response = httpx.get("http://localhost:8041/_memory.json") assert { "database": "_memory", "path": "/_memory", @@ -23,205 +19,11 @@ def test_serve_localhost_http(ds_localhost_http_server): ) def test_serve_unix_domain_socket(ds_unix_domain_socket_server): _, uds = ds_unix_domain_socket_server - transport = httpx2.HTTPTransport(uds=uds) - with httpx2.Client(transport=transport) as client: - response = client.get("http://localhost/_memory.json") + transport = httpx.HTTPTransport(uds=uds) + client = httpx.Client(transport=transport) + response = client.get("http://localhost/_memory.json") assert { "database": "_memory", "path": "/_memory", "tables": [], }.items() <= response.json().items() - - -# Shaped after datasette-litestream's startup hook, which schedules a -# background task with asyncio.get_running_loop().create_task(...): -# https://github.com/datasette/datasette-litestream -MARKER_TASK_PLUGIN = """ -import asyncio -from datasette import hookimpl -from datasette.utils.asgi import Response - - -@hookimpl -def startup(datasette): - datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1 - - async def _mark(): - # Must await before setting the flag: a task with no internal - # await point could finish on the throwaway loop before it - # closed, masking the regression this test guards against. - await asyncio.sleep(0.2) - datasette._marker_task_ran = True - - asyncio.get_running_loop().create_task(_mark()) - - -@hookimpl -def register_routes(): - async def marker_status(datasette): - return Response.json( - { - "marker_task_ran": getattr(datasette, "_marker_task_ran", False), - "startup_calls": getattr(datasette, "_startup_calls", 0), - } - ) - - return [(r"^/-/marker-task-ran$", marker_status)] -""" - - -STARTUP_ERROR_PLUGIN = """ -from datasette import hookimpl -from datasette.utils import StartupError - - -@hookimpl -def startup(datasette): - raise StartupError("boom from plugin") -""" - - -@pytest.mark.serial -def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins): - """ - Litestream-shaped regression test: a startup hook that does - asyncio.get_running_loop().create_task(...) must have that task - actually execute before/while the server is handling requests. This - only holds if invoke_startup() and uvicorn.Server.serve() share one - event loop. This test fails against unmodified main, where - invoke_startup() runs on a throwaway loop that is closed before - uvicorn opens its own loop to serve. - """ - _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN}) - # The fixture has already waited for the server to answer requests. The - # marker task deliberately awaits before setting its flag, so poll for a - # moment rather than assuming it landed before the first request arrived. - deadline = time.time() + 3.0 - payload = {} - while time.time() < deadline: - payload = httpx2.get( - f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0 - ).json() - if payload["marker_task_ran"]: - break - time.sleep(0.05) - assert payload.get("marker_task_ran"), ( - "The startup hook's asyncio.create_task(...) never ran - " - "invoke_startup() and the server are not sharing an event loop" - ) - # Polling above means this test would also pass if the startup hook were - # re-run on the serving loop by the first-request fallback - which would - # hide exactly the bug being tested. invoke_startup() is idempotent today - # so that cannot happen; assert it explicitly so that if the idempotency - # guard is ever removed this test fails loudly instead of silently - # becoming a no-op. - assert payload["startup_calls"] == 1, ( - "startup hook ran {} times - the marker may have been set by a " - "re-run on the serving loop rather than by the original task".format( - payload["startup_calls"] - ) - ) - - -@pytest.mark.serial -def test_startup_error_fails_fast_before_port_binds(serve_with_plugins): - """ - A "startup" plugin hook that raises StartupError must fail fast: print - the message, exit non-zero, and never accept a connection on the port - - the failure must happen before uvicorn.Server binds the socket. - """ - proc, port = serve_with_plugins( - {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False - ) - stdout, _ = proc.communicate(timeout=15) - output = stdout.decode("utf-8") - assert proc.returncode not in (0, None), output - assert "boom from plugin" in output, output - - # Nothing is listening on the port now the process has exited. This - # confirms the socket was not left bound; on its own it cannot prove the - # failure preceded the bind, since a port nothing ever touched also - # refuses connections. - with ( - pytest.raises(OSError), - socket.create_connection(("127.0.0.1", port), timeout=0.2), - ): - pass - - -# Verify that SIGTERM and SIGINT sent to `datasette serve` trigger uvicorn's -# lifespan.shutdown event and run the plugin shutdown hooks. The plugin below -# writes a sentinel file from its shutdown hook so the tests can check that -# cleanup ran after the server subprocess exits. -SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE = """ -import pathlib -from datasette import hookimpl - -SENTINEL_PATH = {sentinel_path!r} - - -@hookimpl -def shutdown(datasette): - pathlib.Path(SENTINEL_PATH).write_text("shutdown ran", "utf-8") -""" - - -def _start_serve_with_shutdown_sentinel(serve_with_plugins, tmp_path): - sentinel_path = tmp_path / "shutdown-sentinel.txt" - proc, _ = serve_with_plugins( - { - "shutdown_sentinel_plugin": SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE.format( - sentinel_path=str(sentinel_path) - ) - } - ) - return proc, sentinel_path - - -@pytest.mark.serial -def test_sigterm_runs_shutdown_hooks(serve_with_plugins, tmp_path): - ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel( - serve_with_plugins, tmp_path - ) - assert not sentinel_path.exists() - ds_proc.send_signal(signal.SIGTERM) - try: - ds_proc.wait(timeout=10) - except subprocess.TimeoutExpired: - ds_proc.kill() - ds_proc.wait() - raise AssertionError( - "datasette serve did not exit within 10s of SIGTERM\n" - + ds_proc.stdout.read().decode("utf-8") - ) - output = ds_proc.stdout.read().decode("utf-8") - assert sentinel_path.exists(), ( - "shutdown hook never wrote its sentinel file after SIGTERM\n" + output - ) - assert sentinel_path.read_text("utf-8") == "shutdown ran" - - -@pytest.mark.serial -@pytest.mark.skipif( - not hasattr(signal, "SIGINT"), reason="Requires signal.SIGINT support" -) -def test_sigint_runs_shutdown_hooks(serve_with_plugins, tmp_path): - ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel( - serve_with_plugins, tmp_path - ) - assert not sentinel_path.exists() - ds_proc.send_signal(signal.SIGINT) - try: - ds_proc.wait(timeout=10) - except subprocess.TimeoutExpired: - ds_proc.kill() - ds_proc.wait() - raise AssertionError( - "datasette serve did not exit within 10s of SIGINT\n" - + ds_proc.stdout.read().decode("utf-8") - ) - output = ds_proc.stdout.read().decode("utf-8") - assert sentinel_path.exists(), ( - "shutdown hook never wrote its sentinel file after SIGINT\n" + output - ) - assert sentinel_path.read_text("utf-8") == "shutdown ran" diff --git a/tests/test_column_types.py b/tests/test_column_types.py index d8dfc627..cd308ec9 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -1,11 +1,7 @@ import json import logging -import time -import markupsafe -import pytest from bs4 import BeautifulSoup as Soup - from datasette.app import Datasette from datasette.column_types import ( ColumnType, @@ -13,7 +9,11 @@ from datasette.column_types import ( ) from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.utils import StartupError, error_body, sqlite3 +from datasette.utils import error_body, sqlite3 +from datasette.utils import StartupError +import markupsafe +import pytest +import time @pytest.fixture @@ -31,7 +31,6 @@ def ds_ct(tmp_path_factory): "'https://example.com', '{\"key\": \"value\"}')" ) db.commit() - db.close() ds = Datasette( [db_path], config={ @@ -71,7 +70,6 @@ def ds_ct_editor_permission(tmp_path_factory): "'https://example.com', '{\"key\": \"value\"}')" ) db.commit() - db.close() ds = Datasette( [db_path], config={ @@ -106,7 +104,7 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", } diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 00540464..42c6ae60 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,12 +1,10 @@ import json import pathlib - import pytest from datasette.app import Datasette -from datasette.utils import StartupError from datasette.utils.sqlite import sqlite3 - +from datasette.utils import StartupError from .fixtures import TestClient as _TestClient PLUGIN = """ @@ -111,7 +109,7 @@ def test_settings(config_dir_client): def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status - plugins = response.json + plugins = response.json["plugins"] assert "hooray.py" in {p["name"] for p in plugins} assert "non_py_file.txt" not in {p["name"] for p in plugins} assert "mypy_cache" not in {p["name"] for p in plugins} diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index ffd0870c..11e53224 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -1,9 +1,7 @@ -import sqlite3 -import urllib - -from click.testing import CliRunner - from datasette.cli import cli +from click.testing import CliRunner +import urllib +import sqlite3 def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled): @@ -42,7 +40,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory): db_dir = tmp_path_factory.mktemp("dbs") dbs = [] for i in range(11): - path = str(db_dir / f"db_{i}.db") + path = str(db_dir / "db_{}.db".format(i)) conn = sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 6c78f69d..2fcfb216 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -44,7 +44,7 @@ async def _run_middleware(scope): await mw(scope, None, send) if inner_called: return ("allowed",) - start = next(m for m in sent if m["type"] == "http.response.start") + start = [m for m in sent if m["type"] == "http.response.start"][0] return ("blocked", start["status"]) diff --git a/tests/test_csv.py b/tests/test_csv.py index adae7e24..a2f03776 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,9 +1,7 @@ -import urllib.parse - -import pytest -from bs4 import BeautifulSoup as Soup - from datasette.app import Datasette +from bs4 import BeautifulSoup as Soup +import pytest +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello @@ -166,66 +164,6 @@ async def test_custom_sql_csv(ds_client): assert response.text == EXPECTED_CUSTOM_CSV -@pytest.mark.asyncio -@pytest.mark.parametrize("download", (False, True)) -@pytest.mark.parametrize( - "query_string,expected_error", - ( - ("sql=select+blah", "no such column: blah"), - ("sql=select+*+from+missing", "no such table: missing"), - ("sql=select+from", 'near "from": syntax error'), - ( - "sql=delete+from+simple_primary_key", - "Statement must be a SELECT", - ), - ("", "?sql= is required"), - ( - "sql=select+sleep(0.01)&_timelimit=5", - ( - "SQL query took too long. The time limit is" - " controlled by the sql_time_limit_ms setting." - ), - ), - ), -) -async def test_custom_sql_csv_errors(ds_client, query_string, expected_error, download): - if download: - query_string += "&_dl=1" - response = await ds_client.get(f"/fixtures/-/query.csv?{query_string}") - assert response.status_code == 400 - assert response.headers["content-type"] == "text/plain; charset=utf-8" - assert "content-disposition" not in response.headers - assert response.text == expected_error - - -@pytest.mark.asyncio -async def test_custom_sql_csv_error_head(ds_client): - response = await ds_client.head("/fixtures/-/query.csv?sql=select+blah") - assert response.status_code == 400 - assert response.headers["content-type"] == "text/plain; charset=utf-8" - assert response.content == b"" - - -@pytest.mark.asyncio -async def test_custom_sql_csv_error_cors(): - ds = Datasette(cors=True) - response = await ds.client.get("/_memory/-/query.csv?sql=select+blah") - assert response.status_code == 400 - assert response.headers["content-type"] == "text/plain; charset=utf-8" - assert response.headers["access-control-allow-origin"] == "*" - assert response.text == "no such column: blah" - - -@pytest.mark.asyncio -async def test_table_csv_error(ds_client): - response = await ds_client.get( - "/fixtures/simple_primary_key.csv?_where=blah&_stream=1" - ) - assert response.status_code == 400 - assert response.headers["content-type"] == "text/plain; charset=utf-8" - assert response.text == "no such column: blah" - - @pytest.mark.asyncio async def test_table_csv_download(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.csv?_dl=1") @@ -289,20 +227,6 @@ async def test_table_csv_stream(ds_client): assert len([b for b in response.content.split(b"\r\n") if b]) == 1002 -@pytest.mark.asyncio -async def test_view_csv_stream(ds_client): - # Without _stream should return header + 100 rows: - response = await ds_client.get("/fixtures/paginated_view.csv?_size=max") - assert len([b for b in response.content.split(b"\r\n") if b]) == 101 - # With _stream=1 should paginate through all pages and return header + 202 rows - response = await ds_client.get("/fixtures/paginated_view.csv?_stream=1") - lines = [b for b in response.content.split(b"\r\n") if b] - assert len(lines) == 203 - # Ensure there are no duplicate rows from looping - assert len(set(lines[1:])) == 202 - assert lines[0] == b"content,content_extra" - - def test_csv_trace(app_client_with_trace): response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1") assert response.headers["content-type"] == "text/html; charset=utf-8" diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 32cfc43d..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -1,7 +1,5 @@ import pathlib - import pytest - from .fixtures import make_app_client TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates") diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index f456a17f..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -1,5 +1,4 @@ import pytest - from datasette.app import Datasette from datasette.resources import DatabaseResource, TableResource diff --git a/tests/test_docs.py b/tests/test_docs.py index b36b773e..0bcb5e62 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -2,22 +2,20 @@ Tests to ensure certain things are documented. """ -import re -from pathlib import Path - -import pytest - -import datasette.fixtures # noqa: F401 from datasette import app, utils +import datasette.fixtures # noqa: F401 from datasette.app import Datasette from datasette.filters import Filters +from pathlib import Path +import pytest +import re docs_path = Path(__file__).parent.parent / "docs" label_re = re.compile(r"\.\. _([^\s:]+):") def get_headings(content, underline="-"): - heading_re = re.compile(rf"(\w+)(\([^)]*\))?\n\{underline}+\n") + heading_re = re.compile(r"(\w+)(\([^)]*\))?\n\{}+\n".format(underline)) return {h[0] for h in heading_re.findall(content)} @@ -27,20 +25,14 @@ def get_labels(filename): @pytest.fixture(scope="session") -def settings_sections(): - content = (docs_path / "settings.rst").read_text() - sections = re.split(r"^(\w+)\n~+\n", content, flags=re.MULTILINE) - return dict(zip(sections[1::2], sections[2::2])) +def settings_headings(): + return get_headings((docs_path / "settings.rst").read_text(), "~") -def test_settings_are_documented(settings_sections, subtests): +def test_settings_are_documented(settings_headings, subtests): for setting in app.SETTINGS: with subtests.test(setting=setting.name): - assert setting.name in settings_sections - assert ( - f'setting_default(cog, "{setting.name}")' - in settings_sections[setting.name] - ) + assert setting.name in settings_headings @pytest.fixture(scope="session") diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index 4a0014b4..613160ac 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -1,10 +1,9 @@ # fmt: off # -- start datasette_with_plugin_fixture -- -import pytest -import pytest_asyncio - from datasette import hookimpl from datasette.app import Datasette +import pytest +import pytest_asyncio @pytest_asyncio.fixture diff --git a/tests/test_error_shape.py b/tests/test_error_shape.py index 94c9a7c9..768814fd 100644 --- a/tests/test_error_shape.py +++ b/tests/test_error_shape.py @@ -17,10 +17,8 @@ present and the legacy "title" key must not be. https://github.com/simonw/datasette/issues - 1.0 API consistency """ -import time - import pytest - +import time from datasette.app import Datasette from datasette.utils import sqlite3 @@ -88,7 +86,7 @@ async def test_write_api_validation_error_shape(ds_error_shape): "/data/docs/-/insert", json={"rows": [{"nope": 1}, {"also_nope": 2}]}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", }, ) @@ -412,7 +410,7 @@ async def test_expired_token_returns_401(ds_error_shape): ) ) response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) data = assert_canonical_error(response, 401) assert "expired" in data["error"].lower() @@ -448,7 +446,7 @@ async def test_valid_token_still_authenticates(ds_error_shape): ) ) response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) assert response.status_code == 200 assert response.json()["actor"]["id"] == "root" @@ -479,7 +477,7 @@ async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory): ds.sign({"a": "root", "t": int(time.time())}, namespace="token") ) response = await ds.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) data = assert_canonical_error(response, 401) assert "not enabled" in data["error"] @@ -644,7 +642,7 @@ async def test_query_list_size_rejects_non_integer(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("endpoint", ("allowed", "rules")) async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint): - base = f"/-/{endpoint}.json?action=view-instance" + base = "/-/{}.json?action=view-instance".format(endpoint) ok = await ds_error_shape.client.get( base + "&_size=1&_page=1", actor={"id": "root"} ) diff --git a/tests/test_extras.py b/tests/test_extras.py index 4e008926..73b4965e 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -1,5 +1,4 @@ import asyncio -from typing import ClassVar import pytest @@ -8,7 +7,7 @@ from datasette.extras import Extra, ExtraRegistry, ExtraScope class SlowValueExtra(Extra): description = "Returns context['value'], optionally slowly" - scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} + scopes = {ExtraScope.TABLE} async def resolve(self, context): if context["slow"]: @@ -18,7 +17,7 @@ class SlowValueExtra(Extra): class DependentExtra(Extra): description = "Depends on slow_value" - scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} + scopes = {ExtraScope.TABLE} async def resolve(self, context, slow_value): return slow_value + 1 @@ -26,7 +25,7 @@ class DependentExtra(Extra): class InternalOnlyExtra(Extra): description = "Internal extra for HTML templates only" - scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} + scopes = {ExtraScope.TABLE} public = False async def resolve(self, context): @@ -53,7 +52,7 @@ def _registered_extra_classes(): @pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key()) def test_registered_extras_have_descriptions(cls): # Every registered extra is part of the documented template/JSON contract - assert cls.description, f"{cls.__name__} is missing a description" + assert cls.description, "{} is missing a description".format(cls.__name__) def test_registry_is_built_once_per_scope(): diff --git a/tests/test_facets.py b/tests/test_facets.py index eccfbb93..8c22ffce 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,15 +1,11 @@ -import json -from urllib.parse import parse_qsl, urlsplit - -import pytest - from datasette.app import Datasette from datasette.database import Database -from datasette.facets import ArrayFacet, ColumnFacet, DateFacet, Facet -from datasette.utils import detect_json1 +from datasette.facets import Facet, ColumnFacet, ArrayFacet, DateFacet from datasette.utils.asgi import Request - +from datasette.utils import detect_json1 from .fixtures import make_app_client +import json +import pytest @pytest.mark.asyncio @@ -152,63 +148,6 @@ async def test_column_facet_results(ds_client): ] == buckets -@pytest.mark.asyncio -@pytest.mark.parametrize( - "column,filters,remaining", - [ - ("COUNTY", "COUNTY=Lee", []), - ("COUNTY", "COUNTY__exact=Lee", []), - ("COUNTY", "COUNTY=Lee&COUNTY__exact=Lee", []), - ("COUNTY", "COUNTY__exact=Lee&COUNTY__exact=Lee", []), - ( - "COUNTY", - "COUNTY__exact=Lee&COUNTY__exact=Polk", - [("COUNTY__exact", "Polk")], - ), - ("_county", "_county__exact=Lee", []), - ("_county", "_county__exact=Lee&_county=Lee", [("_county", "Lee")]), - ], -) -async def test_column_facet_selected_exact_filters( - ds_client, column, filters, remaining -): - facet = ColumnFacet( - ds_client.ds, - Request.fake(f"/?_facet={column}&{filters}&other=keep&_sort={column}"), - database="fixtures", - sql=f"select 'Lee' as {column}", - ) - buckets, timed_out = await facet.facet_results() - assert not timed_out - result = buckets[0]["results"][0] - assert result["selected"] is True - assert parse_qsl(urlsplit(result["toggle_url"]).query) == [ - ("_facet", column), - *remaining, - ("other", "keep"), - ("_sort", column), - ] - - -@pytest.mark.asyncio -async def test_column_facet_underscore_argument_is_not_a_filter(ds_client): - facet = ColumnFacet( - ds_client.ds, - Request.fake("/?_facet=_county&_county=Lee"), - database="fixtures", - sql="select 'Lee' as _county", - ) - buckets, timed_out = await facet.facet_results() - assert not timed_out - result = buckets[0]["results"][0] - assert result["selected"] is False - assert parse_qsl(urlsplit(result["toggle_url"]).query) == [ - ("_facet", "_county"), - ("_county", "Lee"), - ("_county__exact", "Lee"), - ] - - @pytest.mark.asyncio async def test_column_facet_results_column_starts_with_underscore(ds_client): facet = ColumnFacet( @@ -598,7 +537,7 @@ async def test_facet_size(): for j in range(1, 4): await db.execute_write( "insert into neighbourhoods (city, neighbourhood) values (?, ?)", - [f"City {i}", f"Neighbourhood {j}"], + ["City {}".format(i), "Neighbourhood {}".format(j)], ) response = await ds.client.get( "/test_facet_size/neighbourhoods.json?_extra=suggested_facets" diff --git a/tests/test_filters.py b/tests/test_filters.py index 9f201fdf..eda9e9a1 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,7 +1,6 @@ -import pytest - -from datasette.filters import Filters, search_filters, through_filters, where_filters +from datasette.filters import Filters, through_filters, where_filters, search_filters from datasette.utils.asgi import Request +import pytest @pytest.mark.parametrize( @@ -66,12 +65,12 @@ from datasette.utils.asgi import Request # JSON arraycontains, arraynotcontains ( (("Availability+Info__arraycontains", "yes"),), - [':p0 in (select value from json_each("table"."Availability+Info"))'], + [":p0 in (select value from json_each([table].[Availability+Info]))"], ["yes"], ), ( (("Availability+Info__arraynotcontains", "yes"),), - [':p0 not in (select value from json_each("table"."Availability+Info"))'], + [":p0 not in (select value from json_each([table].[Availability+Info]))"], ["yes"], ), ], @@ -83,35 +82,6 @@ def test_build_where(args, expected_where, expected_params): assert {f"p{i}": param for i, param in enumerate(expected_params)} == actual_params -@pytest.mark.parametrize( - "key,expected_where", - ( - ( - 'has"quote__exact', - '"has""quote" = :p0', - ), - ( - 'has"quote__isnull', - '"has""quote" is null', - ), - ( - "has]bracket__arraycontains", - ':p0 in (select value from json_each("table"."has]bracket"))', - ), - ), -) -def test_build_where_escapes_column_names(key, expected_where): - filters = Filters(((key, "value"),)) - sql_bits, _ = filters.build_where_clauses("table") - assert sql_bits == [expected_where] - - -def test_build_where_escapes_table_name(): - filters = Filters((("tags__arraycontains", "value"),)) - sql_bits, _ = filters.build_where_clauses("items]bracket") - assert sql_bits == [':p0 in (select value from json_each("items]bracket"."tags"))'] - - @pytest.mark.asyncio async def test_through_filters_from_request(ds_client): request = Request.fake( diff --git a/tests/test_fts_permissions.py b/tests/test_fts_permissions.py deleted file mode 100644 index 24a40cef..00000000 --- a/tests/test_fts_permissions.py +++ /dev/null @@ -1,412 +0,0 @@ -import pytest - -from datasette import hookimpl -from datasette.app import Datasette -from datasette.permissions import Action, PermissionSQL, _permission_check_cache -from datasette.resources import DatabaseResource, TableResource -from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies - - -@pytest.mark.asyncio -@pytest.mark.parametrize("fts_module", ["fts4", "fts5"]) -@pytest.mark.parametrize("actor", [None, {"id": "root"}], ids=["anonymous", "root"]) -async def test_derived_permissions_allow_one_hop_but_deny_nested_sources( - fts_module, actor -): - class InspectPlugin: - @hookimpl - def register_actions(self): - return [ - Action( - name="inspect-derived", - description="Inspect a table", - resource_class=TableResource, - also_requires="view-table", - ) - ] - - @hookimpl - def permission_resources_sql(self, action): - if action == "inspect-derived": - return PermissionSQL( - sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'inspect allowed' AS reason" - ) - - ds = Datasette(memory=True) - ds.pm.register(InspectPlugin(), name="inspect-derived-test") - db = ds.add_memory_database( - f"derived_one_hop_{fts_module}_{actor is not None}", name="data" - ) - await db.execute_write("create table Documents (body text)") - await db.execute_write( - f"create virtual table Search using {fts_module}(body, content='Documents')" - ) - await db.execute_write( - f"create virtual table Nested using {fts_module}(body, content='sEaRcH')" - ) - await ds.invoke_startup() - token = _permission_check_cache.set({}) - try: - # Both direct permissions are allowed, but a derived source makes its - # dependent unavailable even to an actor who can view the whole chain. - # Check and cache Search first so its cached grant cannot grant Nested. - for table, expected in ( - ("Documents", True), - ("Search", True), - ("Nested", False), - ("Search_docsize", False), - ): - for spelling in (table, table.upper(), table.lower()): - assert await ds.allowed_many( - actions=["view-table", "inspect-derived"], - resource=TableResource("data", spelling), - actor=actor, - ) == {"view-table": expected, "inspect-derived": expected} - - page = await ds.allowed_resources( - "view-table", actor, parent="data", include_is_private=True, limit=1000 - ) - allowed = {resource.child for resource in page.resources} - assert {"Documents", "Search"}.issubset(allowed) - assert "Nested" not in allowed - assert "Search_docsize" not in allowed - finally: - _permission_check_cache.reset(token) - ds.pm.unregister(name="inspect-derived-test") - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("listing", [False, True], ids=["individual", "listing"]) -async def test_derived_permission_discovery_error_is_retried(monkeypatch, listing): - ds = Datasette(memory=True) - db = ds.add_memory_database(f"derived_discovery_error_{listing}", name="data") - await db.execute_write("create table documents (id integer primary key)") - await ds.invoke_startup() - - class UnavailableSchema: - def execute(self, sql): - raise sqlite3.DatabaseError("schema temporarily unavailable") - - async def check(): - if listing: - return await ds.allowed_resources("view-table", parent="data") - return await ds.allowed( - action="view-table", resource=TableResource("data", "documents") - ) - - token = _permission_check_cache.set({}) - try: - with monkeypatch.context() as patch: - patch.setattr( - "datasette.database.sqlite_derived_table_dependencies", - lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()), - ) - with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"): - await check() - - # Failed discovery must not cache an empty map or a permission grant. - assert db._cached_derived_table_dependencies is None - assert not _permission_check_cache.get() - result = await check() - if listing: - assert [resource.child for resource in result.resources] == ["documents"] - else: - assert result is True - assert db._cached_derived_table_dependencies is not None - finally: - _permission_check_cache.reset(token) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("fts_module", ("fts4", "fts5")) -async def test_external_content_fts_inherits_content_table_view_permission(fts_module): - actor = {"id": "reader"} - secret_marker = "ISSUE_17_EXTERNAL_CONTENT_FTS_SECRET" - ds = Datasette( - memory=True, - config={ - "permissions": { - "view-instance": {"id": "reader"}, - "view-database": {"id": "reader"}, - "view-table": {"id": "reader"}, - "execute-sql": {"id": "nobody"}, - }, - "databases": { - "data": { - "tables": { - "secret": {"permissions": {"view-table": False}}, - } - } - }, - }, - ) - db = ds.add_memory_database(f"issue_17_{fts_module}_permissions", name="data") - await db.execute_write("create table secret (id integer primary key, body text)") - await db.execute_write( - "insert into secret (body) values (?)", - [secret_marker], - ) - fts_options = "body, content='secret'" - if fts_module == "fts5": - fts_options += ", content_rowid='id'" - await db.execute_write( - f"create virtual table secret_fts using {fts_module}({fts_options})" - ) - await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')") - await ds.invoke_startup() - - try: - assert "secret_fts" in await db.hidden_table_names() - assert ( - await ds.allowed( - action="execute-sql", - resource=DatabaseResource("data"), - actor=actor, - ) - is False - ) - - direct = await ds.client.get("/data/secret.json", actor=actor) - assert direct.status_code == 403 - - companion = await ds.client.get( - "/data/secret_fts.json?_shape=array", - actor=actor, - ) - assert companion.status_code in (403, 404), ( - "An automatically hidden external-content FTS table must inherit " - "the content table's view denial or be unavailable: " - f"{companion.text}" - ) - assert secret_marker not in companion.text - finally: - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("fts_module", ("fts4", "fts5")) -@pytest.mark.parametrize("contentless", (False, True), ids=("internal", "contentless")) -async def test_fts_shadow_tables_inherit_logical_table_view_permission( - fts_module, contentless -): - table_config = { - "secret_fts": {"permissions": {"view-table": False}}, - # An explicit allow on one implementation table must not override - # the logical FTS table's denial. - "secret_fts_docsize": {"permissions": {"view-table": True}}, - } - ds = Datasette( - memory=True, - config={ - "permissions": { - "view-instance": True, - "view-database": True, - "view-table": True, - "execute-sql": False, - }, - "databases": {"data": {"tables": table_config}}, - }, - ) - db = ds.add_memory_database( - f"issue_17_{fts_module}_{'contentless' if contentless else 'internal'}", - name="data", - ) - options = "body, content=''" if contentless else "body" - await db.execute_write( - f"create virtual table secret_fts using {fts_module}({options})" - ) - await db.execute_write( - "insert into secret_fts(rowid, body) values (1, 'ISSUE_17_SHADOW_SECRET')" - ) - await ds.invoke_startup() - - try: - dependencies = await db.derived_table_dependencies() - shadow_tables = sorted( - table for table, source in dependencies.items() if source == "secret_fts" - ) - assert shadow_tables - assert "secret_fts_docsize" in shadow_tables - - for shadow_table in shadow_tables: - assert ( - await ds.allowed( - action="view-table", - resource=TableResource("data", shadow_table), - ) - is False - ) - response = await ds.client.get(f"/data/{shadow_table}.json?_shape=array") - assert response.status_code == 403 - assert "ISSUE_17_SHADOW_SECRET" not in response.text - - allowed = await ds.allowed_resources("view-table", parent="data", limit=1000) - allowed_names = {resource.child for resource in allowed.resources} - assert not set(shadow_tables).intersection(allowed_names) - - database_json = await ds.client.get("/data.json") - assert database_json.status_code == 200 - for shadow_table in shadow_tables: - assert shadow_table not in database_json.text - - schema_json = await ds.client.get("/data/-/schema.json") - assert schema_json.status_code == 200 - for shadow_table in shadow_tables: - assert shadow_table not in schema_json.text - finally: - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "content_allowed,companion_allowed,expected", - ( - (False, True, False), - (True, False, False), - (True, True, True), - ), -) -async def test_external_content_and_companion_permissions_are_both_required( - content_allowed, companion_allowed, expected -): - ds = Datasette( - memory=True, - default_deny=True, - config={ - "permissions": { - "view-instance": True, - "view-database": True, - }, - "databases": { - "data": { - "tables": { - "secret": {"permissions": {"view-table": content_allowed}}, - "secret_fts": { - "permissions": {"view-table": companion_allowed} - }, - } - } - }, - }, - ) - db = ds.add_memory_database( - f"issue_17_explicit_{int(content_allowed)}_{int(companion_allowed)}", - name="data", - ) - await db.execute_write("create table secret(id integer primary key, body text)") - await db.execute_write("insert into secret(body) values ('ISSUE_17_MATRIX_SECRET')") - await db.execute_write( - "create virtual table secret_fts using fts5(" - "body, content='secret', content_rowid='id')" - ) - await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')") - await ds.invoke_startup() - - try: - assert ( - await ds.allowed( - action="view-table", - resource=TableResource("data", "secret_fts"), - ) - is expected - ) - response = await ds.client.get("/data/secret_fts.json?_shape=array") - assert response.status_code == (200 if expected else 403) - if not expected: - assert "ISSUE_17_MATRIX_SECRET" not in response.text - finally: - ds.close() - - -@pytest.mark.asyncio -async def test_derived_tables_propagate_private_flag_and_route_permissions(): - actor = {"id": "reader"} - ds = Datasette( - memory=True, - config={ - "permissions": { - "view-instance": True, - "view-database": True, - "view-table": True, - }, - "databases": { - "data": { - "tables": { - "secret": {"permissions": {"view-table": {"id": "reader"}}} - } - } - }, - }, - ) - db = ds.add_memory_database("issue_17_private_flag", name="data") - await db.execute_write("create table secret(id integer primary key, body text)") - await db.execute_write("insert into secret(body) values ('PRIVATE')") - await db.execute_write( - "create virtual table secret_fts using fts5(" - "body, content='secret', content_rowid='id')" - ) - await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')") - await ds.invoke_startup() - - try: - actor_page = await ds.allowed_resources( - "view-table", actor, parent="data", include_is_private=True, limit=1000 - ) - actor_resources = { - resource.child: resource for resource in actor_page.resources - } - derived_names = set(await db.derived_table_dependencies()) - assert "secret_fts" in actor_resources - assert actor_resources["secret_fts"].private - # Shadow tables depend on the already-derived external-content FTS - # table, so they remain unavailable even to the permitted reader. - assert not (derived_names - {"secret_fts"}).intersection(actor_resources) - - anonymous_page = await ds.allowed_resources( - "view-table", parent="data", limit=1000 - ) - anonymous_names = {resource.child for resource in anonymous_page.resources} - assert not derived_names.intersection(anonymous_names) - - for path in ( - "/data/secret_fts.json?_facet=body", - "/data/secret_fts.csv", - "/data/secret_fts/-/autocomplete?q=PRIVATE", - "/data/secret_fts/-/schema.json", - ): - denied = await ds.client.get(path) - assert denied.status_code == 403 - allowed = await ds.client.get(path, actor=actor) - assert allowed.status_code == 200 - finally: - ds.close() - - -@pytest.mark.asyncio -async def test_cyclic_derived_table_dependencies_fail_closed(): - ds = Datasette(memory=True) - db = ds.add_memory_database("issue_17_cycle", name="data") - await db.execute_write( - "create virtual table first_fts using fts5(body, content='second_fts')" - ) - await db.execute_write( - "create virtual table second_fts using fts5(body, content='first_fts')" - ) - await ds.invoke_startup() - - try: - for table in ("first_fts", "second_fts"): - assert ( - await ds.allowed( - action="view-table", resource=TableResource("data", table) - ) - is False - ) - - page = await ds.allowed_resources("view-table", parent="data", limit=1000) - allowed_names = {resource.child for resource in page.resources} - assert "first_fts" not in allowed_names - assert "second_fts" not in allowed_names - finally: - ds.close() diff --git a/tests/test_html.py b/tests/test_html.py index 42cca701..b4c47d80 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1,19 +1,16 @@ +from bs4 import BeautifulSoup as Soup +from datasette.app import Datasette +from datasette.utils import allowed_pragmas +from .fixtures import make_app_client +from .utils import assert_footer_links, inner_html import copy import hashlib import json import pathlib +import pytest import re import urllib.parse -import pytest -from bs4 import BeautifulSoup as Soup - -from datasette.app import Datasette -from datasette.utils import allowed_pragmas - -from .fixtures import make_app_client -from .utils import assert_footer_links, inner_html - def test_homepage(app_client_two_attached_databases): response = app_client_two_attached_databases.get("/") @@ -36,10 +33,8 @@ def test_homepage(app_client_two_attached_databases): h2 = soup.select("h2")[0] assert "extra database" == h2.text.strip() counts_p, links_p = h2.find_all_next("p")[:2] - # Shadow tables of the external-content index are denied, so they do not - # contribute to the table or row totals. assert ( - "2 rows in 1 table, 2 rows in 1 hidden table, 1 view" == counts_p.text.strip() + "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip() ) # We should only show visible, not hidden tables here: table_links = [ @@ -147,7 +142,9 @@ def test_static_mounts_hash_cache_control(): ) incorrect_hash = hashlib.sha256(b"incorrect").hexdigest()[:12] - response = client.get(f"/custom-static/test_html.py?_hash={incorrect_hash}") + response = client.get( + "/custom-static/test_html.py?_hash={}".format(incorrect_hash) + ) assert response.status_code == 200 assert "cache-control" not in response.headers @@ -222,9 +219,11 @@ async def test_disallowed_custom_sql_pragma(ds_client): "/fixtures/-/query?sql=SELECT+*+FROM+pragma_not_on_allow_list('idx52')" ) assert response.status_code == 400 - pragmas = ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) + pragmas = ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) assert ( - f"Statement contained a disallowed PRAGMA. Allowed pragma functions are {pragmas}" + "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( + pragmas + ) in response.text ) @@ -779,8 +778,8 @@ def test_stored_query_show_hide_metadata_option( }, memory=True, ) as client: - expected_show_hide_fragment = ( - f'({expected_show_hide_text})' + expected_show_hide_fragment = '({})'.format( + expected_show_hide_link, expected_show_hide_text ) response = client.get("/_memory/one" + querystring) html = response.text @@ -789,7 +788,10 @@ def test_stored_query_show_hide_metadata_option( )[0] assert show_hide_fragment == expected_show_hide_fragment if expected_hidden: - assert f'' in html + assert ( + ''.format(expected_hidden) + in html + ) else: assert '; rel="alternate"; type="application/json+datasette"' + assert link == '<{}>; rel="alternate"; type="application/json+datasette"'.format( + expected + ) assert ( - f'' + ''.format( + expected + ) in response.text ) @@ -1286,8 +1292,8 @@ async def test_database_color(ds_client): expected_color = ds_client.ds.get_database("fixtures").color # Should be something like #9403e5 expected_fragments = ( - f"10px solid #{expected_color}", - f"border-color: #{expected_color}", + "10px solid #{}".format(expected_color), + "border-color: #{}".format(expected_color), ) assert len(expected_color) == 6 for path in ( diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index b4bb964d..e1ab51bb 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,6 +1,5 @@ -import sqlite3 - import pytest +import sqlite3 from datasette.utils import escape_sqlite from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL @@ -138,7 +137,7 @@ async def test_internal_foreign_key_references(ds_client): return { row[1] for row in conn.execute( - f"PRAGMA table_info({escape_sqlite(table_name)})" + "PRAGMA table_info({})".format(escape_sqlite(table_name)) ).fetchall() } @@ -148,7 +147,7 @@ async def test_internal_foreign_key_references(ds_client): for _, name in sorted( (row[5], row[1]) for row in conn.execute( - f"PRAGMA table_info({escape_sqlite(table_name)})" + "PRAGMA table_info({})".format(escape_sqlite(table_name)) ).fetchall() if row[5] ) @@ -160,7 +159,7 @@ async def test_internal_foreign_key_references(ds_client): for table_name in table_names: foreign_key_rows = conn.execute( - f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" + "PRAGMA foreign_key_list({})".format(escape_sqlite(table_name)) ).fetchall() foreign_keys_by_id = {} for foreign_key in foreign_key_rows: @@ -170,16 +169,25 @@ async def test_internal_foreign_key_references(ds_client): foreign_key_rows.sort(key=lambda row: row[1]) other_table = foreign_key_rows[0][2] other_columns = [row[4] for row in foreign_key_rows] - message = f'Column "{table_name}.{foreign_key_rows[0][3]}" references other table "{other_table}" which does not exist' + message = 'Column "{}.{}" references other table "{}" which does not exist'.format( + table_name, foreign_key_rows[0][3], other_table + ) assert other_table in table_names, message + " (bad table)" if all(other_column is None for other_column in other_columns): other_columns = primary_keys_for_table(other_table) - length_message = f'Foreign key from "{table_name}" to "{other_table}" has {len(foreign_key_rows)} columns but references {len(other_columns)} columns' + length_message = 'Foreign key from "{}" to "{}" has {} columns but references {} columns'.format( + table_name, + other_table, + len(foreign_key_rows), + len(other_columns), + ) assert len(other_columns) == len(foreign_key_rows), length_message for foreign_key, other_column in zip(foreign_key_rows, other_columns): column = foreign_key[3] - message = f'Column "{table_name}.{column}" references other column "{other_table}.{other_column}" which does not exist' + message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format( + table_name, column, other_table, other_column + ) assert other_column in columns_by_table[other_table], ( message + " (bad column)" ) @@ -237,10 +245,10 @@ async def test_stale_catalog_entry_database_fix(tmp_path): @pytest.mark.asyncio async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): - import sqlite3 - from datasette.app import Datasette + import sqlite3 + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") bravo_db_path = str(tmp_path / "bravo.db") @@ -285,10 +293,10 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path @pytest.mark.asyncio async def test_orphan_stale_catalog_child_entries_removed(tmp_path): - import sqlite3 - from datasette.app import Datasette + import sqlite3 + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 14a7ce11..bad4e8ca 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -3,28 +3,16 @@ Tests for the datasette.database.Database class """ import asyncio -import uuid from types import SimpleNamespace - -import pytest -import sqlite_utils - from datasette.app import Datasette -from datasette.database import ( - Database, - DatasetteClosedError, - ExecuteWriteResult, - MultipleValues, - QueryInterrupted, - Results, - _deliver_write_result, -) +from datasette.database import Database, ExecuteWriteResult, Results, MultipleValues +from datasette.database import DatasetteClosedError +from datasette.database import _deliver_write_result +from datasette.utils.sqlite import sqlite3, supports_returning from datasette.utils import Column -from datasette.utils.sqlite import ( - sqlite3, - sqlite_derived_table_dependencies, - supports_returning, -) +import pytest +import time +import uuid requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -43,31 +31,6 @@ async def test_execute(db): assert 15 == len(results) -@pytest.mark.asyncio -async def test_derived_dependency_cache_survives_failed_refresh(monkeypatch): - ds = Datasette(memory=True) - db = ds.add_memory_database(uuid.uuid4().hex, name="data") - await db.derived_table_dependencies() - previous_cache = db._cached_derived_table_dependencies - await db.execute_write("create table dependency_cache_refresh (id integer)") - - class UnavailableSchema: - def execute(self, sql): - raise sqlite3.DatabaseError("schema temporarily unavailable") - - with monkeypatch.context() as patch: - patch.setattr( - "datasette.database.sqlite_derived_table_dependencies", - lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()), - ) - with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"): - await db.derived_table_dependencies() - assert db._cached_derived_table_dependencies == previous_cache - - await db.derived_table_dependencies() - assert db._cached_derived_table_dependencies[0] != previous_cache[0] - - @pytest.mark.asyncio async def test_results_first(db): assert None is (await db.execute("select * from facetable where pk > 100")).first() @@ -80,7 +43,7 @@ async def test_results_first(db): @pytest.mark.parametrize("expected", (True, False)) async def test_results_bool(db, expected): where = "" if expected else "where pk = 0" - results = await db.execute(f"select * from facetable {where}") + results = await db.execute("select * from facetable {}".format(where)) assert bool(results) is expected @@ -508,31 +471,6 @@ async def test_view_names(db): ] -@pytest.mark.asyncio -async def test_execute_write_custom_time_limit(): - ds = Datasette(settings={"sql_time_limit_ms": 1}) - db = ds.add_memory_database(uuid.uuid4().hex, name="write_limits") - await ds.invoke_startup() - # Bounded work from PR #51; even without a limit this finishes on its own. - sql = ( - "with recursive c(x) as " - "(select 1 union all select x+1 from c where x < 800000) " - "select x from c where x < 0" - ) - try: - await db.execute_write("create table items(value integer)") - with pytest.raises(QueryInterrupted): - await db.execute(sql) - # Writes take their own explicit limit, independent of the read setting. - with pytest.raises(QueryInterrupted): - await db.execute_write(f"insert into items(value) {sql}", time_limit_ms=1) - # Interruption must leave the connection available for subsequent writes. - await db.execute_write("insert into items(value) values (1)") - assert (await db.execute("select value from items")).single_value() == 1 - finally: - ds.close() - - @pytest.mark.asyncio async def test_execute_write_block_true(db): result = await db.execute_write( @@ -677,7 +615,7 @@ async def test_execute_write_block_false(db): "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], ) - await asyncio.sleep(0.1) + time.sleep(0.1) rows = await db.execute("select name from roadside_attractions where pk = 1") assert "Mystery!" == rows.rows[0][0] @@ -695,7 +633,7 @@ async def test_execute_write_with_returning_block_false(db): ) assert isinstance(task_id, uuid.UUID) - await asyncio.sleep(0.1) + time.sleep(0.1) assert ( await db.execute("select name from write_returning_block_false") ).single_value() == "Cleo" @@ -760,33 +698,6 @@ async def test_execute_write_fn_block_false(db): assert isinstance(task_id, uuid.UUID) -@pytest.mark.asyncio -@pytest.mark.parametrize("disable_threads", (False, True)) -async def test_execute_write_fn_block_false_returns_uuid(tmp_path, disable_threads): - # block=False is documented to return "a UUID representing the queued task". - # With num_sql_threads=0 there is no write thread, so the non-threaded branch - # has to satisfy the same contract as the threaded one. - settings = {"num_sql_threads": 0} if disable_threads else {} - ds = Datasette([], memory=True, settings=settings) - await ds.invoke_startup() - db = ds.add_memory_database("test_block_false") - await db.execute_write( - "create table if not exists t (id integer primary key, v text)" - ) - - def write_fn(conn): - conn.execute("insert into t (v) values ('a')") - # Returns None, like most write functions. - - task_id = await db.execute_write_fn(write_fn, block=False) - - assert isinstance(task_id, uuid.UUID) - # Distinct per call, so a caller can tell two queued tasks apart. - second = await db.execute_write_fn(write_fn, block=False) - assert isinstance(second, uuid.UUID) - assert second != task_id - - @pytest.mark.asyncio async def test_execute_write_fn_block_true(db): def write_fn(conn): @@ -807,51 +718,15 @@ async def test_execute_write_fn_exception(db): await db.execute_write_fn(write_fn) -@pytest.mark.asyncio -@pytest.mark.parametrize("num_sql_threads", (0, 1)) -async def test_execute_write_fn_sqlite_utils_transaction(tmp_path, num_sql_threads): - # A write inside a failing Datasette task must never become visible or - # survive the rollback. Exercise both the synchronous and writer-thread - # paths against a file-backed database so a second connection can observe - # committed state independently. - db_path = tmp_path / "test.db" - sqlite3.connect(db_path).close() - ds = Datasette([str(db_path)], settings={"num_sql_threads": num_sql_threads}) - db = ds.get_database("test") - await db.execute_write("create table items (id integer primary key)") - # This reader is used inside the write callback, which may run on another - # thread, but it is never accessed concurrently. - reader = sqlite3.connect(db_path, check_same_thread=False) - - def insert_then_fail(conn): - # Datasette must open the outer transaction before sqlite-utils writes. - assert conn.in_transaction - sqlite_utils.Database(conn)["items"].insert({"id": 1}) - # If sqlite-utils committed its own transaction, this would return 1. - assert reader.execute("select count(*) from items").fetchone()[0] == 0 - # Simulate a later step failing after the sqlite-utils write succeeded. - raise ValueError("deliberate") - - try: - with pytest.raises(ValueError, match="deliberate"): - await db.execute_write_fn(insert_then_fail) - # The outer transaction must roll back the sqlite-utils write as well. - assert reader.execute("select count(*) from items").fetchone()[0] == 0 - finally: - reader.close() - db.close() - - @pytest.mark.asyncio @pytest.mark.parametrize("param_name", ["conn", "connection", "db", "c"]) async def test_execute_write_fn_accepts_any_single_param_name(db, param_name): # Plugins historically relied on the fact that the callback was invoked # positionally, so any parameter name worked. Preserve that contract. scope = {} - # exec() is how we build a function with a parameterized argument name - exec( # noqa: S102 - f"def write_fn({param_name}):\n" - f" return {param_name}.execute('select 1 + 1').fetchone()[0]", + exec( + "def write_fn({0}):\n" + " return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name), scope, ) write_fn = scope["write_fn"] @@ -875,9 +750,7 @@ async def test_execute_write_fn_with_track_event(db): @pytest.mark.asyncio -# func_only so the budget covers the write-thread call under test, not the -# one-off app_client fixture setup this test may be first to trigger -@pytest.mark.timeout(1, func_only=True) +@pytest.mark.timeout(1) async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") conn = sqlite3.connect(path) @@ -1305,17 +1178,3 @@ async def test_database_close_is_idempotent(tmpdir): # Second call should be a no-op, not raise db.close() ds._internal_database.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("num_sql_threads", [0, 2]) -@pytest.mark.parametrize("named", [False, True]) -async def test_close_releases_memory_connections(num_sql_threads, named): - ds = Datasette(memory=True, settings={"num_sql_threads": num_sql_threads}) - db = ds.add_memory_database(uuid.uuid4().hex) if named else ds.get_database() - read_connection = await db.execute_fn(lambda conn: conn) - write_connection = await db.execute_write_fn(lambda conn: conn) - ds.close() - for conn in (read_connection, write_connection): - with pytest.raises(sqlite3.ProgrammingError, match="closed"): - conn.execute("select 1") diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index ed2aeaf0..85598c05 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -9,15 +9,13 @@ import importlib import os import sqlite3 import time - -import pytest -from itsdangerous import BadSignature - from datasette import Context -from datasette.app import Database, Datasette, ResourcesSQL +from datasette.app import Datasette, Database, ResourcesSQL from datasette.database import DatasetteClosedError from datasette.resources import DatabaseResource from datasette.utils import PrefixedUrlString +from itsdangerous import BadSignature +import pytest @pytest.fixture @@ -79,7 +77,9 @@ async def test_static_template_function_hashes_core_asset(tmp_path, monkeypatch) template = ds.get_jinja_environment().from_string("{{ static('demo.js') }}") expected_hash = hashlib.sha256(b"const demo = true;").hexdigest()[:12] - assert await template.render_async() == f"/-/static/demo.js?_hash={expected_hash}" + assert await template.render_async() == "/-/static/demo.js?_hash={}".format( + expected_hash + ) assert isinstance(ds.static("demo.js"), PrefixedUrlString) @@ -101,7 +101,7 @@ def test_static_hash_recalculated_when_cache_headers_disabled(tmp_path, monkeypa asset_path.write_bytes(b"let a = 2;") expected_hash = hashlib.sha256(b"let a = 2;").hexdigest()[:12] - assert ds.static("demo.js") == f"/-/static/demo.js?_hash={expected_hash}" + assert ds.static("demo.js") == "/-/static/demo.js?_hash={}".format(expected_hash) assert ds.static("demo.js") != first_url @@ -114,12 +114,12 @@ def test_static_hashes_mounted_static_file(tmp_path): expected_hash = hashlib.sha256(b"body { color: black; }").hexdigest()[:12] assert ds.static("styles.css", mount="assets") == ( - f"/assets/styles.css?_hash={expected_hash}" + "/assets/styles.css?_hash={}".format(expected_hash) ) ds._settings["base_url"] = "/prefix/" assert ds.static("styles.css", mount="assets") == ( - f"/prefix/assets/styles.css?_hash={expected_hash}" + "/prefix/assets/styles.css?_hash={}".format(expected_hash) ) @@ -144,7 +144,9 @@ def test_static_hashes_plugin_static_file(tmp_path, monkeypatch): expected_hash = hashlib.sha256(b"console.log('plugin');").hexdigest()[:12] assert ds.static("plugin.js", plugin="datasette_cluster_map") == ( - f"/-/static-plugins/datasette_cluster_map/plugin.js?_hash={expected_hash}" + "/-/static-plugins/datasette_cluster_map/plugin.js?_hash={}".format( + expected_hash + ) ) diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 29046dc3..e9aaaae8 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,7 +1,6 @@ -import httpx2 +import httpx import pytest import pytest_asyncio - from datasette.app import Datasette @@ -43,7 +42,7 @@ async def datasette_with_permissions(): async def test_client_methods(datasette, method, path, expected_status): client_method = getattr(datasette.client, method) response = await client_method(path) - assert isinstance(response, httpx2.Response) + assert isinstance(response, httpx.Response) assert response.status_code == expected_status # Try that again using datasette.client.request response2 = await datasette.client.request(method, path) @@ -63,7 +62,7 @@ async def test_client_post(datasette, prefix): "message": "A message", }, ) - assert isinstance(response, httpx2.Response) + assert isinstance(response, httpx.Response) assert response.status_code == 302 assert "ds_messages" in response.cookies finally: @@ -135,7 +134,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me response = await client_method("/test_db.json", skip_permission_checks=True) # We don't check status code since some methods might not be allowed, # but we verify the request doesn't fail due to permissions - assert isinstance(response, httpx2.Response) + assert isinstance(response, httpx.Response) @pytest.mark.asyncio @@ -239,7 +238,7 @@ async def test_in_client_returns_false_outside_request(datasette): @pytest.mark.asyncio async def test_in_client_returns_true_inside_request(): """Test that datasette.in_client() returns True inside a client request""" - from datasette import Response, hookimpl + from datasette import hookimpl, Response class TestPlugin: __name__ = "test_in_client_plugin" @@ -340,7 +339,7 @@ async def test_actor_parameter_all_http_methods(datasette, method): client_method = getattr(datasette.client, method) # Just verify no TypeError about unexpected 'actor' kwarg response = await client_method("/", actor={"id": "root"}) - assert isinstance(response, httpx2.Response) + assert isinstance(response, httpx.Response) @pytest.mark.asyncio diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index 91ef368f..6d2dc70a 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -1,8 +1,6 @@ -import json - -import pytest - from datasette.utils.asgi import PayloadTooLarge, Request +import json +import pytest def _post_scope(headers=None): @@ -35,51 +33,6 @@ def _receive_chunks(chunks): return receive -@pytest.mark.parametrize( - "header_name", [b"content-type", b"Content-Type", b"CONTENT-TYPE"] -) -@pytest.mark.parametrize("lookup", ["content-type", "Content-Type", "CONTENT-TYPE"]) -def test_request_headers_case_insensitive(header_name, lookup): - request = Request({"headers": [(header_name, b"application/json")]}, None) - assert request.headers.get(lookup) == "application/json" - assert request.headers[lookup] == "application/json" - assert lookup in request.headers - - -def test_request_headers_mapping(): - request = Request( - { - "headers": [ - (b"Content-Type", b"application/json"), - (b"X-Title", "café".encode("latin-1")), - (b"CONTENT-TYPE", b"text/plain"), - ] - }, - None, - ) - headers = request.headers - expected = {"content-type": "text/plain", "x-title": "café"} - assert headers == expected - assert dict(headers) == expected - assert list(headers) == list(expected) - assert list(headers.keys()) == list(expected.keys()) - assert list(headers.items()) == list(expected.items()) - assert json.loads(json.dumps(headers)) == expected - assert headers["Content-Type"] == "text/plain" - assert headers["X-Title"] == "café" - - -@pytest.mark.parametrize("scope", [{}, {"headers": None}, {"headers": []}]) -def test_request_headers_missing(scope): - headers = Request(scope, None).headers - assert headers == {} - assert headers.get("Content-Type") is None - assert headers.get("Content-Type", "default") == "default" - assert "Content-Type" not in headers - with pytest.raises(KeyError): - headers["Content-Type"] - - @pytest.mark.asyncio async def test_request_post_vars(): scope = { diff --git a/tests/test_internals_response.py b/tests/test_internals_response.py index aa3e1ae2..2366dcde 100644 --- a/tests/test_internals_response.py +++ b/tests/test_internals_response.py @@ -1,8 +1,6 @@ -import json - -import pytest - from datasette.utils.asgi import Response +import json +import pytest def test_response_html(): diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 50c61995..24fa745d 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -1,7 +1,6 @@ -import pytest - from datasette.app import Datasette from datasette.utils import PrefixedUrlString +import pytest @pytest.fixture(scope="module") diff --git a/tests/test_label_column_for_table.py b/tests/test_label_column_for_table.py index b67b8882..7667b595 100644 --- a/tests/test_label_column_for_table.py +++ b/tests/test_label_column_for_table.py @@ -1,7 +1,6 @@ import pytest - -from datasette.app import Datasette from datasette.database import Database +from datasette.app import Datasette @pytest.mark.asyncio diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py deleted file mode 100644 index 3a25ccbc..00000000 --- a/tests/test_lifespan.py +++ /dev/null @@ -1,344 +0,0 @@ -""" -Tests for wiring Datasette startup (setup_db table counts + invoke_startup) -into the ASGI lifespan protocol. - -These exercise Datasette._startup_sequence() via three different callers: -- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request) -- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan - events (this is what DatasetteClient / plain httpx2.ASGITransport uses) -- Both at once, to prove startup hooks run at most once -""" - -import asyncio -import contextlib -import sqlite3 - -import httpx2 -import pytest - -from datasette import hookimpl -from datasette.app import Datasette -from datasette.database import Database -from datasette.plugins import pm - - -async def _drive_lifespan_startup(app): - """Send a single lifespan.startup message into app's ASGI lifespan loop - and return the list of messages sent back - without ever sending - lifespan.shutdown. Mirrors what a real server does: after startup - completes it parks waiting for the next event. We cancel that wait - once we've observed the startup response, rather than closing the - Datasette instance down with a shutdown message. - """ - messages_sent = [] - startup_responded = asyncio.Event() - delivered = False - - async def receive(): - nonlocal delivered - if not delivered: - delivered = True - return {"type": "lifespan.startup"} - # No further messages: block until the task is cancelled below, - # same as a real server parked waiting for lifespan.shutdown. - await asyncio.Event().wait() - - async def send(message): - messages_sent.append(message) - startup_responded.set() - - task = asyncio.create_task(app({"type": "lifespan"}, receive, send)) - try: - await asyncio.wait_for(startup_responded.wait(), timeout=5) - finally: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - return messages_sent - - -@pytest.mark.asyncio -async def test_lifespan_startup_runs_before_any_request(): - ds = Datasette(memory=True) - assert ds._startup_invoked is False - app = ds.app() - - messages = await _drive_lifespan_startup(app) - - assert {"type": "lifespan.startup.complete"} in messages - assert ds._startup_invoked is True - # Internal catalog tables should be populated too, entirely without an - # HTTP request having been made. - internal_db = ds.get_internal_database() - databases = await internal_db.execute("select * from catalog_databases") - assert len(databases.rows) >= 1 - - -@pytest.mark.asyncio -async def test_lifespan_startup_failure_reports_lifespan_startup_failed(): - class RaisingStartupPlugin: - __name__ = "RaisingStartupPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - raise RuntimeError("boom from startup hook") - - return inner - - ds = Datasette(memory=True) - pm.register(RaisingStartupPlugin(), name="raising_startup_plugin") - try: - app = ds.app() - messages = await _drive_lifespan_startup(app) - finally: - pm.unregister(name="raising_startup_plugin") - - assert messages == [ - {"type": "lifespan.startup.failed", "message": "boom from startup hook"} - ] - # The exception happened before invoke_startup() got to the end of its - # body, so startup is not considered to have completed. - assert ds._startup_invoked is False - - -@pytest.mark.asyncio -async def test_startup_runs_exactly_once_across_lifespan_and_first_request(): - call_count = {"n": 0} - - class CountingStartupPlugin: - __name__ = "CountingStartupPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - call_count["n"] += 1 - - return inner - - ds = Datasette(memory=True) - pm.register(CountingStartupPlugin(), name="counting_startup_plugin") - try: - # Build the ASGI app once, the way a real deployment does - and - # reuse the SAME app instance for both the lifespan drive and the - # HTTP requests below, since a fresh ds.app() call would reset the - # AsgiRunOnFirstRequest fallback's state. - app = ds.app() - - messages = await _drive_lifespan_startup(app) - assert {"type": "lifespan.startup.complete"} in messages - assert call_count["n"] == 1 - - # A first HTTP request (as if the host never sent lifespan events, - # or lifespan already ran) should not run the hook again. - transport = httpx2.ASGITransport(app=app) - async with httpx2.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - response1 = await client.get("/-/versions.json") - assert response1.status_code == 200 - # ... nor should a second, repeat request. - response2 = await client.get("/-/versions.json") - assert response2.status_code == 200 - finally: - pm.unregister(name="counting_startup_plugin") - - assert call_count["n"] == 1 - - -@pytest.mark.asyncio -async def test_no_lifespan_first_request_still_triggers_startup(): - # Pin today's behavior: a client that never drives ASGI lifespan events - # at all (like httpx2.ASGITransport, which DatasetteClient uses) still - # gets startup armed by the AsgiRunOnFirstRequest fallback. - ds = Datasette(memory=True) - assert ds._startup_invoked is False - app = ds.app() - transport = httpx2.ASGITransport(app=app) - async with httpx2.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - response = await client.get("/-/versions.json") - assert response.status_code == 200 - - assert ds._startup_invoked is True - internal_db = ds.get_internal_database() - databases = await internal_db.execute("select * from catalog_databases") - assert len(databases.rows) >= 1 - - -@pytest.mark.asyncio -async def test_datasette_client_first_request_triggers_startup(): - # Same as above, but through the real DatasetteClient (ds.client) that - # plugins and tests actually use, to confirm nothing regressed there. - ds = Datasette(memory=True) - assert ds._startup_invoked is False - response = await ds.client.get("/-/versions.json") - assert response.status_code == 200 - assert ds._startup_invoked is True - - -@pytest.mark.asyncio -async def test_concurrent_first_requests_all_wait_for_slow_startup(): - call_count = {"n": 0} - - class SlowStartupPlugin: - __name__ = "SlowStartupPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - call_count["n"] += 1 - await asyncio.sleep(0.2) - - return inner - - ds = Datasette(memory=True) - pm.register(SlowStartupPlugin(), name="slow_startup_plugin") - try: - app = ds.app() - transport = httpx2.ASGITransport(app=app) - async with httpx2.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - responses = await asyncio.gather( - *[client.get("/-/versions.json") for _ in range(10)] - ) - finally: - pm.unregister(name="slow_startup_plugin") - - # Every one of the 10 simultaneous first requests must have blocked - # until startup actually finished, not raced ahead of it. - assert all(response.status_code == 200 for response in responses) - assert call_count["n"] == 1 - assert ds._startup_invoked is True - - -@pytest.mark.asyncio -async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch): - # Regression test: `datasette serve` (cli.py _serve_async) calls - # ds.invoke_startup() directly, before uvicorn ever sends a - # lifespan.startup event that drives _startup_sequence(). If - # _startup_sequence()'s fast path only checked `_startup_invoked`, it - # would see startup already done and skip the immutable-database - # table-count precompute (setup_db) entirely - a silent regression - # versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally - # on request #1. - db_path = tmp_path / "immutable.db" - conn = sqlite3.connect(str(db_path)) - conn.execute("create table t (id integer primary key)") - conn.commit() - conn.close() - - ds = Datasette([], immutables=[str(db_path)]) - - call_count = {"n": 0} - original_table_counts = Database.table_counts - - async def counting_table_counts(self, *args, **kwargs): - call_count["n"] += 1 - return await original_table_counts(self, *args, **kwargs) - - monkeypatch.setattr(Database, "table_counts", counting_table_counts) - - # Simulate the CLI path: invoke_startup() runs directly and completes - # BEFORE _startup_sequence() ever gets a chance to run setup_db. - await ds.invoke_startup() - assert ds._startup_invoked is True - assert call_count["n"] == 0 - - # The lifespan/first-request path (or the CLI itself, per the fix) - # calling the shared entry point afterwards must still precompute - # table counts for immutable databases. - await ds._startup_sequence() - assert call_count["n"] == 1 - assert ds._setup_db_done is True - - # Idempotency: a second call must not recompute. - await ds._startup_sequence() - assert call_count["n"] == 1 - - -@pytest.mark.asyncio -async def test_asgi_wrapper_runs_after_startup_fallback_path(): - class AssertStartupPlugin: - __name__ = "AssertStartupPlugin" - - @hookimpl - def asgi_wrapper(self, datasette): - def wrap(app): - async def check_startup(scope, receive, send): - if scope["type"] == "http": - assert ( - datasette._startup_invoked is True - ), "asgi_wrapper saw an http scope before startup completed" - await app(scope, receive, send) - - return check_startup - - return wrap - - ds = Datasette(memory=True) - pm.register(AssertStartupPlugin(), name="assert_startup_plugin") - try: - assert ds._startup_invoked is False - app = ds.app() - transport = httpx2.ASGITransport(app=app) - async with httpx2.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - response = await client.get("/-/versions.json") - assert response.status_code == 200 - finally: - pm.unregister(name="assert_startup_plugin") - - assert ds._startup_invoked is True - - -@pytest.mark.asyncio -async def test_short_circuit_wrapper_no_longer_defers_startup(): - # Middleware that returns a response before getting to the rest of - # Datasette should still cause _startup_invoked=True - class ShortCircuitPlugin: - __name__ = "ShortCircuitPlugin" - - @hookimpl - def asgi_wrapper(self, datasette): - def wrap(app): - async def forbidden(scope, receive, send): - if scope["type"] != "http": - await app(scope, receive, send) - return - await send( - { - "type": "http.response.start", - "status": 403, - "headers": [[b"content-type", b"text/plain"]], - } - ) - await send( - { - "type": "http.response.body", - "body": b"Forbidden", - } - ) - - return forbidden - - return wrap - - ds = Datasette(memory=True) - pm.register(ShortCircuitPlugin(), name="short_circuit_plugin") - try: - assert ds._startup_invoked is False - app = ds.app() - transport = httpx2.ASGITransport(app=app) - async with httpx2.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - response = await client.get("/-/versions.json") - assert response.status_code == 403 - finally: - pm.unregister(name="short_circuit_plugin") - - assert ds._startup_invoked is True diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py index a7c2bc24..cdadb091 100644 --- a/tests/test_load_extensions.py +++ b/tests/test_load_extensions.py @@ -1,9 +1,6 @@ -from pathlib import Path -from unittest import mock - -import pytest - from datasette.app import Datasette +import pytest +from pathlib import Path # not necessarily a full path - the full compiled path looks like "ext.dylib" # or another suffix, but sqlite will, under the hood, decide which file @@ -21,29 +18,6 @@ def has_compiled_ext(): return False -@pytest.mark.parametrize("load_fails", (False, True)) -def test_load_extension_is_disabled(load_fails): - ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH]) - connection = mock.Mock() - if load_fails: - connection.load_extension.side_effect = RuntimeError - - if load_fails: - with pytest.raises(RuntimeError): - ds._prepare_connection(connection, "data") - else: - ds._prepare_connection(connection, "data") - - # Extensions are loaded using the Python API, never via SQL - assert connection.load_extension.mock_calls == [ - mock.call(COMPILED_EXTENSION_PATH), - ] - assert connection.enable_load_extension.mock_calls == [ - mock.call(True), - mock.call(False), - ] - - @pytest.mark.asyncio @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") async def test_load_extension_default_entrypoint(): @@ -88,20 +62,3 @@ async def test_load_extension_multiple_entrypoints(): response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()") assert response.status_code == 200 assert response.json()["rows"][0][0] == "c" - - -@pytest.mark.asyncio -@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") -async def test_sql_cannot_load_additional_extension(): - ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH]) - - response = await ds.client.get( - "/_memory/-/query.json", - params={ - "sql": "select load_extension(:path, :entrypoint)", - "path": COMPILED_EXTENSION_PATH, - "entrypoint": "sqlite3_ext_b_init", - }, - ) - assert response.status_code == 400 - assert response.json()["error"] == "not authorized" diff --git a/tests/test_messages.py b/tests/test_messages.py index 60eb7938..62d9f647 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -1,6 +1,5 @@ -import pytest - from .utils import cookie_was_deleted +import pytest @pytest.mark.asyncio diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 8cc91db3..0dc3ecd7 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -4,145 +4,14 @@ Tests for request.form() multipart form data parsing. Uses TDD approach - these tests are written first, then implementation follows. """ -import asyncio import base64 import json -import threading +import pytest from collections import namedtuple -import pytest from multipart_form_data_conformance import get_tests_dir -from datasette.utils.asgi import BadRequest, Request -from datasette.utils.multipart import MultipartParseError, parse_form_data - - -@pytest.fixture -def upload_files(monkeypatch): - from datasette.utils import multipart - - files = [] - original = multipart.tempfile.SpooledTemporaryFile - - def create_file(*args, **kwargs): - file = original(*args, **kwargs) - files.append(file) - return file - - monkeypatch.setattr(multipart.tempfile, "SpooledTemporaryFile", create_file) - yield files - for file in files: - file.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "failure", - [ - "file_limit", - "request_limit", - "truncated", - "disconnect", - "receive_error", - "cancel", - ], -) -async def test_failed_upload_closes_completed_and_partial_files(upload_files, failure): - # Complete one file and begin another, flushing the parser's 64 KiB batch. - body = ( - b'--boundary\r\nContent-Disposition: form-data; name="one"; filename="one"\r\n\r\n' - b'first\r\n--boundary\r\nContent-Disposition: form-data; name="two"; filename="two"\r\n\r\n' - + b"x" * (64 * 1024) - ) - calls = 0 - - async def receive(): - nonlocal calls - calls += 1 - if calls == 1: - return {"type": "http.request", "body": body, "more_body": True} - if failure == "disconnect": - return {"type": "http.disconnect"} - if failure == "receive_error": - raise OSError("receive failed") - if failure == "cancel": - raise asyncio.CancelledError - return {"type": "http.request", "body": b"x" * 100, "more_body": False} - - kwargs = {} - if failure == "file_limit": - kwargs["max_file_size"] = 1024 - if failure == "request_limit": - kwargs["max_request_size"] = len(body) - error = { - "receive_error": OSError, - "cancel": asyncio.CancelledError, - }.get(failure, MultipartParseError) - with pytest.raises(error): - await parse_form_data( - receive, "multipart/form-data; boundary=boundary", files=True, **kwargs - ) - assert len(upload_files) == 2 - assert all(file.closed for file in upload_files) - - -@pytest.mark.asyncio -async def test_unnamed_upload_is_closed(upload_files): - body = ( - b'--boundary\r\nContent-Disposition: form-data; filename="ignored"\r\n\r\n' - b"content\r\n--boundary--\r\n" - ) - form = await parse_form_data( - make_receive(body), "multipart/form-data; boundary=boundary", files=True - ) - assert len(form) == 0 - assert len(upload_files) == 1 - assert upload_files[0].closed - - -@pytest.mark.asyncio -@pytest.mark.parametrize("worker_error", [False, True]) -async def test_cancelled_upload_waits_for_worker( - upload_files, monkeypatch, worker_error -): - from datasette.utils.multipart import MultipartParser - - loop = asyncio.get_running_loop() - started = asyncio.Event() - release = threading.Event() - original_feed = MultipartParser.feed - - def blocking_feed(self, chunk): - original_feed(self, chunk) - loop.call_soon_threadsafe(started.set) - assert release.wait(5) - if worker_error: - raise MultipartParseError("worker failed") - - monkeypatch.setattr(MultipartParser, "feed", blocking_feed) - body = ( - b'--boundary\r\nContent-Disposition: form-data; name="file"; filename="file"\r\n\r\n' - + b"x" * (64 * 1024) - ) - task = asyncio.create_task( - parse_form_data( - make_receive(body), "multipart/form-data; boundary=boundary", files=True - ) - ) - try: - await asyncio.wait_for(started.wait(), timeout=5) - # A second cancellation must also leave cleanup waiting for the worker. - for _ in range(2): - task.cancel() - await asyncio.sleep(0) - assert not task.done() - assert len(upload_files) == 1 - assert not upload_files[0].closed - finally: - release.set() - with pytest.raises(asyncio.CancelledError): - await task - assert upload_files[0].closed +from datasette.utils.asgi import Request, BadRequest def make_receive(body: bytes): @@ -1209,73 +1078,75 @@ async def test_conformance(test_spec, headers, body): await request.form(files=True) return - async with await request.form(files=True) as form: - # Verify each expected part - for i, expected_part in enumerate(expected["parts"]): - name = expected_part["name"] + # Parse form data + form = await request.form(files=True) - # Get value(s) for this name - values = form.getlist(name) + # Verify each expected part + for i, expected_part in enumerate(expected["parts"]): + name = expected_part["name"] - # Find the value at the correct index for this name - # (handles multiple values with same name) - same_name_count = sum(1 for p in expected["parts"][:i] if p["name"] == name) + # Get value(s) for this name + values = form.getlist(name) - if same_name_count >= len(values): - pytest.fail( - f"Expected part {name} at index {same_name_count} but only {len(values)} found" - ) + # Find the value at the correct index for this name + # (handles multiple values with same name) + same_name_count = sum(1 for p in expected["parts"][:i] if p["name"] == name) - value = values[same_name_count] - - # Determine expected content - if "body_base64" in expected_part: - expected_content = base64.b64decode(expected_part["body_base64"]) - elif "body_text" in expected_part: - expected_content = expected_part["body_text"].encode("utf-8") - else: - expected_content = None - - # Check for file vs field - # A part is a file if it has a filename OR filename_star - is_file = ( - expected_part.get("filename") is not None - or expected_part.get("filename_star") is not None + if same_name_count >= len(values): + pytest.fail( + f"Expected part {name} at index {same_name_count} but only {len(values)} found" ) - if is_file: - # It's a file - assert hasattr(value, "filename"), f"Expected file for {name}" + value = values[same_name_count] - # Check filename - use filename_star if present, else filename - expected_filename = expected_part.get( - "filename_star" - ) or expected_part.get("filename") - if expected_filename: - assert ( - value.filename == expected_filename - ), f"Filename mismatch: expected {expected_filename!r}, got {value.filename!r}" + # Determine expected content + if "body_base64" in expected_part: + expected_content = base64.b64decode(expected_part["body_base64"]) + elif "body_text" in expected_part: + expected_content = expected_part["body_text"].encode("utf-8") + else: + expected_content = None - if expected_part.get("content_type"): - assert value.content_type == expected_part["content_type"] + # Check for file vs field + # A part is a file if it has a filename OR filename_star + is_file = ( + expected_part.get("filename") is not None + or expected_part.get("filename_star") is not None + ) - content = await value.read() + if is_file: + # It's a file + assert hasattr(value, "filename"), f"Expected file for {name}" + + # Check filename - use filename_star if present, else filename + expected_filename = expected_part.get("filename_star") or expected_part.get( + "filename" + ) + if expected_filename: assert ( - len(content) == expected_part["body_size"] - ), f"Size mismatch: expected {expected_part['body_size']}, got {len(content)}" - if expected_content is not None: - assert content == expected_content - else: - # It's a text field - if hasattr(value, "filename"): - pytest.fail(f"Expected text field for {name}, got file") + value.filename == expected_filename + ), f"Filename mismatch: expected {expected_filename!r}, got {value.filename!r}" - if expected_content is not None: - # For text fields, value is a string - try: - expected_text = expected_content.decode("utf-8") - except UnicodeDecodeError: - expected_text = expected_content.decode("latin-1") - assert ( - value == expected_text - ), f"Value mismatch: expected {expected_text!r}, got {value!r}" + if expected_part.get("content_type"): + assert value.content_type == expected_part["content_type"] + + content = await value.read() + assert ( + len(content) == expected_part["body_size"] + ), f"Size mismatch: expected {expected_part['body_size']}, got {len(content)}" + if expected_content is not None: + assert content == expected_content + else: + # It's a text field + if hasattr(value, "filename"): + pytest.fail(f"Expected text field for {name}, got file") + + if expected_content is not None: + # For text fields, value is a string + try: + expected_text = expected_content.decode("utf-8") + except UnicodeDecodeError: + expected_text = expected_content.decode("latin-1") + assert ( + value == expected_text + ), f"Value mismatch: expected {expected_text!r}, got {value!r}" diff --git a/tests/test_numeric_filter_values.py b/tests/test_numeric_filter_values.py deleted file mode 100644 index 1fceeabe..00000000 --- a/tests/test_numeric_filter_values.py +++ /dev/null @@ -1,47 +0,0 @@ -import sqlite3 -from contextlib import closing - -import pytest - -from datasette.filters import Filters - - -@pytest.mark.parametrize( - "value,expected", - ( - ("3.5", 3.5), - ("-2", -2), - ("-2.5", -2.5), - ("1e3", 1000.0), - ("not-a-number", "not-a-number"), - ("nan", "nan"), - ("inf", "inf"), - ("-inf", "-inf"), - ), -) -def test_numeric_filter_parameters(value, expected): - filters = Filters((("score__gt", value),)) - sql_bits, params = filters.build_where_clauses("items") - - assert sql_bits == ['"score" > :p0'] - assert params == {"p0": expected} - - -def test_numeric_filter_parameters_against_calculated_view(): - with closing(sqlite3.connect(":memory:")) as conn: - conn.execute("create table searchable(pk integer)") - conn.executemany("insert into searchable(pk) values (?)", [(0,), (1,), (2,)]) - conn.execute( - "create view calculated as " - "select pk + 1 as pk_plus_one, pk / 2.0 as score from searchable" - ) - - sql_bits, params = Filters((("score__gt", "0.1"),)).build_where_clauses( - "calculated" - ) - rows = conn.execute( - "select score from calculated where {}".format(" and ".join(sql_bits)), - params, - ).fetchall() - - assert rows == [(0.5,), (1.0,)] diff --git a/tests/test_package.py b/tests/test_package.py index 43b20589..f05f3ece 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,11 +1,9 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import os import pathlib -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli class CaptureDockerfile: diff --git a/tests/test_permission_endpoints.py b/tests/test_permission_endpoints.py index 774064fd..8726ab62 100644 --- a/tests/test_permission_endpoints.py +++ b/tests/test_permission_endpoints.py @@ -6,7 +6,6 @@ Tests for permission endpoints: import pytest import pytest_asyncio - from datasette.app import Datasette @@ -433,8 +432,8 @@ async def test_execute_sql_requires_view_database(): A user who has execute-sql permission but not view-database permission should not be able to execute SQL on that database. """ - from datasette import hookimpl from datasette.permissions import PermissionSQL + from datasette import hookimpl class TestPermissionPlugin: __name__ = "TestPermissionPlugin" @@ -494,31 +493,3 @@ async def test_execute_sql_requires_view_database(): ) finally: ds.pm.unregister(plugin) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("path", ["/-/allowed", "/-/allowed.json?action=view-table"]) -async def test_allowed_requires_view_instance(path): - """ - GHSA-hp2x-vx2r-6vxg: /-/allowed should be gated like its /-/rules sibling. - - An actor who is denied view-instance gets 403 from / and /-/rules, but - /-/allowed (HTML and JSON) currently returns 200 to the same actor. - """ - ds = Datasette(config={"allow": {"id": "alice"}}) - await ds.invoke_startup() - db = ds.add_memory_database("live") - await db.execute_write("CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY)") - await ds.refresh_schemas() - - assert (await ds.client.get("/")).status_code == 403 - assert (await ds.client.get("/-/rules.json?action=view-table")).status_code == 403 - - response = await ds.client.get(path) - assert response.status_code == 403 - - # Alice is still allowed - response = await ds.client.get( - path, cookies={"ds_actor": ds.client.actor_cookie({"id": "alice"})} - ) - assert response.status_code == 200 diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0a77bd37..88fe577f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,23 +1,20 @@ import collections -import copy -import json -import re -import time -import urllib -from pprint import pprint - -import pytest -import pytest_asyncio from asgiref.sync import async_to_sync -from bs4 import BeautifulSoup as Soup -from click.testing import CliRunner - from datasette.app import Datasette from datasette.cli import cli from datasette.default_permissions import restrictions_allow_action from datasette.utils import UNSTABLE_API_MESSAGE - from .fixtures import assert_permissions_checked, make_app_client +from click.testing import CliRunner +from bs4 import BeautifulSoup as Soup +import copy +import json +from pprint import pprint +import pytest_asyncio +import pytest +import re +import time +import urllib @pytest.fixture(scope="module") @@ -460,20 +457,6 @@ async def test_permissions_debug(ds_client, filter_): assert checks == expected_checks -@pytest.mark.asyncio -@pytest.mark.parametrize( - "permissions_debug,expected_status", - ( - (1, 200), - (0, 403), - ), -) -async def test_permissions_debug_numeric_boolean(permissions_debug, expected_status): - ds = Datasette(config={"permissions": {"permissions-debug": permissions_debug}}) - response = await ds.client.get("/-/permissions") - assert response.status_code == expected_status - - @pytest.mark.asyncio @pytest.mark.parametrize( "actor,allow,expected_fragment", @@ -520,7 +503,6 @@ def view_instance_client(): "/-/plugins", "/-/settings", "/-/threads", - "/-/tasks", "/-/databases", "/-/permissions", "/-/messages", @@ -606,7 +588,9 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta ) assert ( response.status == expected_status - ), f"path: {path}, permissions: {permissions}, expected_status: {expected_status}, status: {response.status}" + ), "path: {}, permissions: {}, expected_status: {}, status: {}".format( + path, permissions, expected_status, response.status + ) finally: cascade_app_client.ds.config = previous_config @@ -764,12 +748,7 @@ async def test_actor_restricted_permissions( } if actor.get("id"): expected["actor_id"] = actor["id"] - data = response.json() - for key, value in expected.items(): - assert data[key] == value - assert data["actor"] == actor - assert data["explanation"]["allowed"] is expected_result - assert data["explanation"]["summary"] + assert response.json() == expected PermConfigTestCase = collections.namedtuple( @@ -1755,8 +1734,6 @@ async def test_permission_check_view_requires_debug_permission(): data = response.json() assert data["action"] == "view-instance" assert data["allowed"] is True - assert data["explanation"]["allowed"] is True - assert data["explanation"]["summary"] @pytest.mark.asyncio @@ -1782,211 +1759,6 @@ async def test_permission_check_view_query_actions(action): } -@pytest.mark.asyncio -async def test_permission_check_explains_specificity_for_hypothetical_actor(): - ds = Datasette( - config={ - "permissions": {"view-table": {"id": "alice"}}, - "databases": { - "analytics": { - "permissions": {"view-table": False}, - "tables": { - "public": {"permissions": {"view-table": {"id": "alice"}}} - }, - } - }, - } - ) - ds.root_enabled = True - await ds.invoke_startup() - - def path_for(child): - return "/-/check.json?" + urllib.parse.urlencode( - { - "action": "view-table", - "parent": "analytics", - "child": child, - "actor": json.dumps({"id": "alice"}), - } - ) - - public_response = await ds.client.get(path_for("public"), actor={"id": "root"}) - assert public_response.status_code == 200 - public = public_response.json() - assert public["actor"] == {"id": "alice"} - assert public["allowed"] is True - assert public["explanation"]["allowed"] is True - assert public["explanation"]["winning_scope"] == "resource" - public_rules = public["explanation"]["matched_rules"] - assert any( - rule["scope"] == "resource" and rule["effect"] == "allow" and rule["decisive"] - for rule in public_rules - ) - assert any( - rule["scope"] == "parent" - and rule["effect"] == "deny" - and rule["ignored_because"] == "A more specific rule matched" - for rule in public_rules - ) - - private_response = await ds.client.get(path_for("private"), actor={"id": "root"}) - assert private_response.status_code == 200 - private = private_response.json() - assert private["allowed"] is False - assert private["explanation"]["allowed"] is False - assert private["explanation"]["winning_scope"] == "parent" - assert private["explanation"]["summary"].startswith("Denied by a parent-level rule") - - -@pytest.mark.asyncio -async def test_permission_check_explains_deny_wins_at_same_scope(): - ds = Datasette(config={"permissions": {"view-table": {"id": "someone-else"}}}) - ds.root_enabled = True - await ds.invoke_startup() - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "view-table", - "parent": "analytics", - "child": "users", - "actor": json.dumps({"id": "alice"}), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - assert data["explanation"]["winning_scope"] == "global" - rules = data["explanation"]["matched_rules"] - assert any(rule["effect"] == "deny" and rule["decisive"] for rule in rules) - assert any( - rule["effect"] == "allow" - and rule["ignored_because"] == "A deny rule matched at the same scope" - for rule in rules - ) - - -@pytest.mark.asyncio -async def test_permission_check_explains_default_deny(): - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "insert-row", - "parent": "analytics", - "child": "users", - "actor": json.dumps({"id": "alice"}), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - explanation = data["explanation"] - assert explanation["allowed"] is False - assert explanation["matched_rules"] == [] - assert explanation["winning_scope"] is None - assert explanation["summary"] == ( - "Denied because no permission rule matched this actor and resource." - ) - - -@pytest.mark.asyncio -async def test_permission_check_explains_actor_restrictions(): - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - restricted_actor = { - "id": "alice", - "_r": {"r": {"analytics": {"public": ["vt"]}}}, - } - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "view-table", - "parent": "analytics", - "child": "private", - "actor": json.dumps(restricted_actor), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - explanation = data["explanation"] - assert explanation["rule_allowed"] is True - assert explanation["restriction_allowed"] is False - assert explanation["allowed"] is False - assert explanation["restrictions"] - assert any( - restriction["allowed"] is False for restriction in explanation["restrictions"] - ) - assert "actor's restrictions" in explanation["summary"] - - -@pytest.mark.asyncio -async def test_permission_check_explains_required_actions(): - from datasette import hookimpl - from datasette.permissions import PermissionSQL - - class StoreQueryPermissions: - @hookimpl - def permission_resources_sql(self, actor, action): - if not actor or actor.get("id") != "alice": - return None - if action == "store-query": - return PermissionSQL( - sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'alice can store queries' AS reason" - ) - if action == "execute-sql": - return PermissionSQL( - sql="SELECT 'analytics' AS parent, NULL AS child, 0 AS allow, 'alice cannot execute SQL' AS reason" - ) - - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - ds.pm.register(StoreQueryPermissions(), name="store-query-test") - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "store-query", - "parent": "analytics", - "actor": json.dumps({"id": "alice"}), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - explanation = data["explanation"] - assert explanation["rule_allowed"] is True - assert explanation["required_actions"][0]["action"] == "execute-sql" - assert explanation["required_actions"][0]["allowed"] is False - assert explanation["summary"] == ( - "Denied because store-query also requires execute-sql, which was denied." - ) - - -@pytest.mark.asyncio -async def test_permission_check_hypothetical_actor_validation(): - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - - response = await ds.client.get( - "/-/check.json?action=view-instance&actor=not-json", - actor={"id": "root"}, - ) - assert response.status_code == 400 - assert response.json()["error"].startswith("Invalid actor JSON:") - - response = await ds.client.get( - "/-/check.json?action=view-instance&actor=%5B%5D", - actor={"id": "root"}, - ) - assert response.status_code == 400 - assert response.json()["error"] == "actor must be a JSON object or null" - - @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ @@ -2041,7 +1813,7 @@ async def test_databases_json_respects_view_database(tmp_path_factory): paths = [] for name in ("public", "private"): - path = str(db_directory / f"{name}.db") + path = str(db_directory / "{}.db".format(name)) conn = _sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 07a11947..ab9d3568 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -5,7 +5,7 @@ import subprocess import sys import time -import httpx2 +import httpx import pytest from datasette.fixtures import write_fixture_database @@ -34,11 +34,11 @@ def wait_for_server(process, url, timeout=30): f"stderr:\n{stderr}" ) try: - response = httpx2.get(url, timeout=1.0) + response = httpx.get(url, timeout=1.0) if response.status_code < 500: return last_error = f"HTTP {response.status_code}: {response.text[:200]}" - except httpx2.HTTPError as ex: + except httpx.HTTPError as ex: last_error = repr(ex) time.sleep(0.1) if process.poll() is None: @@ -108,11 +108,6 @@ def write_playwright_database(db_path): conn = sqlite3.connect(db_path) try: conn.executescript(""" - create table count_numbers (id integer primary key); - with recursive sequence(id) as ( - select 1 union all select id + 1 from sequence where id < 10002 - ) - insert into count_numbers select id from sequence; create table projects ( id integer primary key, title text not null, @@ -341,7 +336,7 @@ def project_rows(datasette_server, **filters): "_shape": "objects", **{key: str(value) for key, value in filters.items()}, } - response = httpx2.get(f"{datasette_server}data/projects.json", params=params) + response = httpx.get(f"{datasette_server}data/projects.json", params=params) response.raise_for_status() return response.json()["rows"] @@ -353,7 +348,7 @@ def project_row(datasette_server, pk): def binary_file_blob(datasette_server, pk): - response = httpx2.get( + response = httpx.get( f"{datasette_server}data/binary_files/{pk}.blob", params={"_blob_column": "data"}, ) @@ -374,7 +369,7 @@ def bulk_default_rows(datasette_server, **filters): "_shape": "objects", **{key: str(value) for key, value in filters.items()}, } - response = httpx2.get(f"{datasette_server}data/bulk_defaults.json", params=params) + response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params) response.raise_for_status() return response.json()["rows"] @@ -384,7 +379,7 @@ def upsert_item_rows(datasette_server, **filters): "_shape": "objects", **{key: str(value) for key, value in filters.items()}, } - response = httpx2.get(f"{datasette_server}data/upsert_items.json", params=params) + response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params) response.raise_for_status() return response.json()["rows"] @@ -415,6 +410,29 @@ def test_datasette_homepage_contains_datasette(page, datasette_server): assert "Datasette" in page.locator("body").inner_text() +@pytest.mark.playwright +def test_table_header_stays_visible_while_scrolling(page, datasette_server): + page.set_viewport_size({"width": 600, "height": 400}) + page.goto(f"{datasette_server}fixtures/sortable?_size=100") + + wrapper = page.locator(".table-wrapper") + header = wrapper.locator("table.rows-and-columns th").first + assert wrapper.evaluate("node => node.scrollWidth > node.clientWidth") + + wrapper.evaluate("""node => window.scrollTo( + 0, + node.getBoundingClientRect().top + window.scrollY + 100 + )""") + page.wait_for_function("""() => Math.abs(document.querySelector( + 'table.rows-and-columns th' + ).getBoundingClientRect().top) < 1""") + assert abs(header.bounding_box()["y"]) < 1 + + wrapper.evaluate("node => node.scrollLeft = 100") + assert wrapper.evaluate("node => node.scrollLeft") == 100 + assert abs(header.bounding_box()["y"]) < 1 + + @pytest.mark.playwright def test_create_table_flow(page, datasette_server): page.goto(f"{datasette_server}data") @@ -478,7 +496,7 @@ def test_create_table_flow(page, datasette_server): page.wait_for_url("**/data/playwright_created") assert "playwright_created" in page.locator("h1").inner_text() - response = httpx2.get( + response = httpx.get( f"{datasette_server}data/playwright_created.json?_extra=columns,column_types" ) response.raise_for_status() @@ -492,7 +510,7 @@ def test_create_table_flow(page, datasette_server): assert data["column_types"] == { "metadata": {"type": "json", "config": None}, } - schema_response = httpx2.get( + schema_response = httpx.get( f"{datasette_server}data/-/query.json", params={ "sql": ( @@ -608,7 +626,7 @@ def test_create_table_from_data_flow(page, datasette_server): dialog.locator(".table-create-save").click() page.wait_for_url("**/data/playwright_from_data") - response = httpx2.get( + response = httpx.get( f"{datasette_server}data/playwright_from_data.json?_shape=objects" ) response.raise_for_status() @@ -644,7 +662,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank( dialog.locator(".table-create-save").click() page.wait_for_url("**/data/playwright_numeric_blanks") - response = httpx2.get( + response = httpx.get( f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects" ) response.raise_for_status() @@ -653,7 +671,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank( {"name": "B", "score": None}, ] - schema_response = httpx2.get( + schema_response = httpx.get( f"{datasette_server}data/-/query.json", params={ "sql": ( @@ -861,7 +879,7 @@ def test_alter_table_flow(page, datasette_server): columns = [] for _ in range(20): - response = httpx2.get(f"{datasette_server}data/projects.json?_extra=columns") + response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns") response.raise_for_status() columns = response.json()["columns"] if "status" in columns: @@ -1033,14 +1051,15 @@ def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): dialog.locator(".table-alter-add-column").click() dialog.locator(".table-alter-column-name").last.fill("escape_me") page.keyboard.press("Escape") - page.wait_for_function("window.__discardConfirmMessages.length === 1") assert page.evaluate("() => window.__discardConfirmMessages") == [ "Discard table changes?" ] assert dialog.evaluate("node => node.open") is True page.evaluate("() => window.__discardConfirmMessages = []") - page.mouse.click(2, 2) + dialog.evaluate( + """node => node.dispatchEvent(new MouseEvent("click", {bubbles: true}))""" + ) assert page.evaluate("() => window.__discardConfirmMessages") == [ "Discard table changes?" ] @@ -1083,92 +1102,6 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins( page.wait_for_url("**/-/playwright-agent") -@pytest.mark.playwright -def test_navigation_search_created_from_javascript(page, datasette_server): - from playwright.sync_api import expect - - page.goto(datasette_server) - page.evaluate("""() => { - const search = document.createElement('navigation-search'); - search.id = 'additional-search'; - search.setAttribute('items', JSON.stringify([ - {name: 'Projects', url: '/data/projects'} - ])); - document.body.append(search); - const unrelated = document.createElement('div'); - unrelated.className = 'search-container'; - unrelated.id = 'outside-search'; - document.body.append(unrelated); - search.openMenu(); - }""") - search = page.locator("#additional-search") - dialog = search.get_by_role("dialog", name="Jump to", exact=True) - expect(dialog).to_be_visible() - # Page styles and ordinary DOM queries can reach the component's controls. - page.add_style_tag( - content="#additional-search .search-input { border-top-color: rgb(1, 2, 3); }" - ) - field = dialog.get_by_role("combobox", name="Jump to", exact=True) - expect(field).to_have_css("border-top-color", "rgb(1, 2, 3)") - assert field.evaluate("node => document.getElementById(node.id) === node") - expect(page.locator("#outside-search")).to_have_css("display", "block") - field.fill("projects") - expect(dialog.get_by_role("option")).to_contain_text("Projects") - field.press("Enter") - page.wait_for_url("**/data/projects") - - -@pytest.mark.playwright -def test_column_chooser_selection_and_drag_in_document(page, datasette_server): - from playwright.sync_api import expect - - page.goto(datasette_server + "data/projects") - page.emulate_media(reduced_motion="reduce") - page.evaluate("""() => { - const chooser = document.createElement('column-chooser'); - chooser.id = 'additional-chooser'; - document.body.append(chooser); - window.appliedColumns = null; - chooser.open({ - columns: ['title', 'notes', 'score'], - selected: ['title', 'notes'], - onApply: columns => { window.appliedColumns = columns; } - }); - }""") - chooser = page.locator("#additional-chooser") - dialog = chooser.get_by_role("dialog", name="Choose columns") - expect(dialog).to_be_visible() - assert dialog.evaluate("""node => { - const id = node.getAttribute('aria-labelledby'); - return document.querySelectorAll(`#${id}`).length === 1 && - node.contains(document.getElementById(id)); - }""") - expect(dialog.locator(".modal-meta")).to_have_text("2 of 3 selected") - dialog.get_by_role("button", name="Deselect all", exact=True).click() - expect(dialog.locator(".modal-meta")).to_have_text("0 of 3 selected") - dialog.get_by_role("button", name="Select all", exact=True).click() - expect(dialog.locator(".modal-meta")).to_have_text("3 of 3 selected") - # Move title after score using the same pointer events as mouse/touch dragging. - handle = dialog.locator(".drag-handle").first.bounding_box() - target = dialog.locator(".drag-item").last.bounding_box() - page.mouse.move(handle["x"] + handle["width"] / 2, handle["y"] + 24) - page.mouse.down() - page.mouse.move(target["x"] + 24, target["y"] + target["height"] - 4, steps=5) - expect(dialog.locator(".drag-ghost")).to_be_visible() - page.mouse.up() - expect(dialog.locator(".drag-item-label")).to_have_text(["notes", "score", "title"]) - dialog.get_by_role("button", name="Apply", exact=True).click() - expect(dialog).not_to_be_visible() - assert page.evaluate("appliedColumns") == ["notes", "score", "title"] - chooser.evaluate( - "node => node.open({columns: ['title', 'notes'], selected: ['title']})" - ) - dialog.get_by_role("button", name="Deselect all", exact=True).click() - dialog.get_by_role("button", name="Cancel", exact=True).click() - expect(dialog).not_to_be_visible() - assert page.evaluate("appliedColumns") == ["notes", "score", "title"] - - @pytest.mark.playwright def test_insert_row_flow_uses_custom_column_field(page, datasette_server): page.add_init_script(""" @@ -1689,323 +1622,8 @@ def test_delete_row_flow_removes_row(page, datasette_server): dialog = page.locator("#row-delete-dialog") dialog.wait_for() assert "Delete row 1" in dialog.inner_text() - dialog.locator(".row-delete-confirm").press("Enter") + dialog.locator(".row-delete-confirm").click() page.locator(".row-mutation-status", has_text="Deleted row 1").wait_for() page.locator('tr[data-row="1"]').wait_for(state="detached") assert project_rows(datasette_server, id=1) == [] - - -@pytest.mark.playwright -def test_count_all(page, datasette_server): - page.goto(datasette_server + "data/count_numbers?id__gt=1&_sort=id") - assert page.locator(".table-count").inner_text() == "10,000+ rows" - with page.expect_response("**/count_numbers/-/count?*") as response: - page.get_by_role("button", name="count all", exact=True).click() - assert response.value.request.method == "POST" - assert response.value.request.post_data is None - assert "content-type" not in response.value.request.headers - assert response.value.json() == {"ok": True, "count": 10001} - page.wait_for_function( - 'document.querySelector(".table-count").textContent === "10,001 rows"' - ) - assert page.locator(".count-all").count() == 0 - assert "id" in page.locator("h3").first.inner_text() - - -@pytest.mark.playwright -def test_count_all_error_retry(page, datasette_server): - page.goto(datasette_server + "data/count_numbers?id__gt=1") - page.route( - "**/count_numbers/-/count?*", - lambda route: route.fulfill( - status=400, - content_type="application/json", - body=json.dumps({"ok": False, "errors": ["Count query timed out"]}), - ), - ) - button = page.get_by_role("button", name="count all", exact=True) - button.click() - page.wait_for_function( - 'document.querySelector(".count-error").textContent === "Count query timed out"' - ) - assert button.is_enabled() - page.unroute("**/count_numbers/-/count?*") - button.click() - page.wait_for_function( - 'document.querySelector(".table-count").textContent === "10,001 rows"' - ) - assert page.locator(".count-error").inner_text() == "" - - -@pytest.mark.playwright -def test_modal_lifecycle(page, datasette_server): - from playwright.sync_api import expect - - page.goto(datasette_server) - page.evaluate( - """() => { - const trigger = document.createElement('button'); - trigger.id = 'modal-trigger'; - trigger.textContent = 'Open test modal'; - document.body.append(trigger); - window.testModal = DatasetteModal.create(); - const dialog = testModal.dialog; - dialog.id = 'test-modal'; - dialog.setAttribute('aria-labelledby', 'test-modal-title'); - dialog.innerHTML = ` -

      Test modal

      - - - `; - // Padding is part of the dialog, never a backdrop dismissal. - dialog.style.padding = '30px'; - document.body.append(testModal); - window.closeSources = []; - testModal.beforeClose = source => { - closeSources.push(source); - return window.allowClose; - }; - window.allowClose = false; - trigger.onclick = () => testModal.show({ - returnFocusTo: trigger, initialFocus: dialog.querySelector('input') - }); - dialog.querySelector('button').onclick = () => testModal.requestClose('cancel'); - }""", - ) - trigger = page.locator("#modal-trigger") - trigger.click() - dialog = page.get_by_role("dialog", name="Test modal", exact=True) - expect(dialog.get_by_role("textbox", name="First field")).to_be_focused() - assert dialog.evaluate("node => node instanceof HTMLDialogElement") - expect(dialog).to_have_css("display", "flex") - # Native modality keeps background content inert and keyboard focus inside. - page.keyboard.press("Tab") - expect(dialog.get_by_role("textbox", name="Second field")).to_be_focused() - page.keyboard.press("Shift+Tab") - expect(dialog.get_by_role("textbox", name="First field")).to_be_focused() - trigger.evaluate("node => node.focus()") - expect(dialog.get_by_role("textbox", name="First field")).to_be_focused() - - page.keyboard.down("Escape") - assert page.evaluate("closeSources") == [] - page.keyboard.up("Escape") - page.wait_for_function("closeSources.length === 1") - assert page.evaluate("closeSources") == ["escape"] - expect(dialog).to_be_visible() - - dialog.click(position={"x": 3, "y": 3}) - assert page.evaluate("closeSources") == ["escape"] - # A drag which starts inside and ends on the backdrop must not dismiss. - box = dialog.bounding_box() - page.mouse.move(box["x"] + 3, box["y"] + 3) - page.mouse.down() - page.mouse.move(2, 2) - page.mouse.up() - assert page.evaluate("closeSources") == ["escape"] - page.mouse.click(2, 2) - assert page.evaluate("closeSources") == ["escape", "backdrop"] - - page.evaluate("testModal.busy = true; allowClose = true") - expect(dialog).to_have_attribute("aria-busy", "true") - page.keyboard.press("Escape") - page.mouse.click(2, 2) - dialog.get_by_role("button", name="Cancel").click() - expect(dialog).to_be_visible() - assert page.evaluate("closeSources") == ["escape", "backdrop"] - page.evaluate("testModal.busy = false") - dialog.get_by_role("button", name="Cancel").click() - expect(dialog).not_to_be_visible() - expect(trigger).to_be_focused() - assert page.evaluate("closeSources") == ["escape", "backdrop", "cancel"] - - # Reopening, including an extra show() call, preserves the original return-focus target. - trigger.click() - page.evaluate("testModal.show()") - page.keyboard.press("Escape") - expect(dialog).not_to_be_visible() - expect(trigger).to_be_focused() - - # Completion bypasses busy/confirmation and must not steal a caller's focus. - trigger.click() - page.evaluate("""() => new Promise(resolve => { - const next = document.createElement('button'); - next.id = 'after-save'; - next.textContent = 'Next action'; - document.body.append(next); - testModal.dialog.addEventListener('close', resolve, {once: true}); - testModal.busy = true; - testModal.close({restoreFocus: false}); - next.focus(); - })""") - expect(dialog).not_to_be_visible() - expect(page.locator("#after-save")).to_be_focused() - - -@pytest.mark.playwright -def test_modal_nested_escape_and_cleanup(page, datasette_server): - from playwright.sync_api import expect - - page.goto(datasette_server + "data/projects") - trigger = page.locator('tr[data-row="1"] button[data-row-action="edit"]') - trigger.click() - dialog = page.locator("#row-edit-dialog") - field = dialog.locator('input[name="title"]') - expect(field).to_be_visible() - field.fill("Unsaved title") - page.evaluate("""() => { - window.confirmations = []; - window.confirm = message => { confirmations.push(message); return false; }; - }""") - # Plugin controls can consume Escape without closing their containing form. - field.evaluate("""node => node.addEventListener('keydown', event => { - if (event.key === 'Escape') event.preventDefault(); - }, {once: true})""") - field.press("Escape") - assert page.evaluate("confirmations") == [] - expect(dialog).to_be_visible() - field.press("Escape") - page.wait_for_function("confirmations.length === 1") - assert page.evaluate("confirmations") == ["Discard unsaved changes to this row?"] - - # A nested native modal closes independently, then returns focus to its field. - field.evaluate("""node => { - node.focus(); - window.nestedModal = DatasetteModal.create(); - nestedModal.dialog.setAttribute('aria-label', 'Nested picker'); - nestedModal.dialog.innerHTML = ''; - node.closest('dialog').append(nestedModal); - nestedModal.show(); - }""") - nested = page.get_by_role("dialog", name="Nested picker") - page.keyboard.press("Escape") - expect(nested).not_to_be_visible() - expect(dialog).to_be_visible() - expect(field).to_be_focused() - assert page.evaluate("confirmations.length") == 1 - - # Closing before keyup cancels the pending confirmation, including on reopen. - page.keyboard.down("Escape") - dialog.locator(".row-edit-cancel").click() - expect(dialog).not_to_be_visible() - trigger.click() - page.keyboard.up("Escape") - expect(field).to_be_visible() - assert page.evaluate("confirmations.length") == 1 - expect(dialog).to_be_visible() - # Native cancel (e.g. an accessibility action) does not wait for keyboard input. - field.fill("Another edit") - dialog.evaluate( - "node => node.dispatchEvent(new Event('cancel', {cancelable: true}))" - ) - assert page.evaluate("confirmations.length") == 2 - dialog.locator(".row-edit-cancel").click() - expect(trigger).to_be_focused() - - -@pytest.mark.playwright -@pytest.mark.parametrize("name", ["jump", "columns", "type", "mobile"]) -def test_modal_consumers_dismiss_and_restore_focus(page, datasette_server, name): - from playwright.sync_api import expect - - page_errors = [] - page.on("pageerror", lambda error: page_errors.append(str(error))) - if name == "mobile": - page.set_viewport_size({"width": 390, "height": 844}) - page.emulate_media(reduced_motion="reduce") - page.goto(datasette_server + "data/projects") - if name == "jump": - trigger = page.locator("details.nav-menu summary") - trigger.click() - page.locator("[data-navigation-search-open]").click() - dialog = page.locator("navigation-search dialog") - elif name == "columns": - # Open through its public API with a real, focused page control. - trigger = page.locator("details.actions-menu-links summary") - trigger.focus() - page.evaluate( - "document.querySelector('column-chooser').open({columns: ['id', 'title'], selected: ['id']})" - ) - dialog = page.locator("column-chooser dialog") - elif name == "type": - trigger = page.locator("details.actions-menu-links summary") - trigger.focus() - page.evaluate( - "openSetColumnTypeDialog(document.querySelector('th[data-column=title]'))" - ) - dialog = page.locator("#set-column-type-dialog") - else: - trigger = page.locator(".column-actions-mobile") - trigger.click() - dialog = page.locator("#mobile-column-actions-dialog") - expect(dialog).to_be_visible() - expect(dialog).to_have_css("border-radius", "8px" if name == "mobile" else "12px") - expect(dialog).to_have_css("animation-name", "none") - assert dialog.evaluate("node => node.parentElement.localName") == "datasette-modal" - assert dialog.evaluate("node => node.getRootNode() === document") - page.keyboard.press("Escape") - expect(dialog).not_to_be_visible() - expect(trigger).to_be_focused() - assert page_errors == [] - - -@pytest.mark.playwright -def test_modal_disconnect_cleans_up_pending_escape(page, datasette_server): - from playwright.sync_api import expect - - page.goto(datasette_server) - page.evaluate("""() => { - window.detachable = DatasetteModal.create(); - detachable.dialog.setAttribute('aria-label', 'Detachable'); - detachable.dialog.innerHTML = ''; - window.closeAttempts = 0; - detachable.beforeClose = () => { closeAttempts++; return false; }; - document.body.append(detachable); - detachable.show(); - }""") - dialog = page.get_by_role("dialog", name="Detachable") - page.keyboard.down("Escape") - page.evaluate("detachable.remove()") - page.keyboard.up("Escape") - assert page.evaluate("closeAttempts") == 0 - assert page.evaluate("detachable.dialog.open") is False - page.evaluate("document.body.append(detachable); detachable.show()") - expect(dialog).to_be_visible() - page.keyboard.press("Escape") - page.wait_for_function("closeAttempts === 1") - expect(dialog).to_be_visible() - - -@pytest.mark.playwright -@pytest.mark.parametrize("kind", ["create", "alter"]) -def test_schema_modal_escape_confirmation_and_focus(page, datasette_server, kind): - from playwright.sync_api import expect - - path = "data" if kind == "create" else "data/projects" - page.goto(datasette_server + path) - menu = page.locator("details.actions-menu-links") - menu.locator("summary").click() - selector = "data-database-action" if kind == "create" else "data-table-action" - menu.locator(f'button[{selector}="{kind}-table"]').click() - dialog = page.locator(f"#table-{kind}-dialog") - if kind == "create": - dialog.locator('input[name="table"]').fill("unsaved_table") - else: - dialog.locator(".table-alter-add-column").click() - # Real browser confirms, including WebKit, should appear once and stay usable. - confirmations = [] - - def reject(prompt): - confirmations.append(prompt.message) - prompt.dismiss() - - page.on("dialog", reject) - with page.expect_event("dialog"): - page.keyboard.press("Escape") - expect(dialog).to_be_visible() - assert len(confirmations) == 1 - page.remove_listener("dialog", reject) - page.on("dialog", lambda prompt: prompt.accept()) - page.keyboard.press("Escape") - expect(dialog).not_to_be_visible() - expect(menu.locator("summary")).to_be_focused() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1084d270..5c4034db 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,3 +1,21 @@ +from bs4 import BeautifulSoup as Soup +from .fixtures import ( + make_app_client, + TEMP_PLUGIN_SECRET_FILE, + PLUGINS_DIR, + TestClient as _TestClient, +) # noqa +from click.testing import CliRunner +from datasette.app import Datasette +from datasette import cli, hookimpl +from datasette.fixtures import TABLES +from datasette.filters import FilterArguments +from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm +from datasette.permissions import PermissionSQL, Action +from datasette.resources import DatabaseResource +from datasette.utils.sqlite import sqlite3 +from datasette.utils import StartupError, await_me_maybe +from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime import importlib @@ -6,31 +24,8 @@ import os import pathlib import re import textwrap -import urllib - import pytest -from bs4 import BeautifulSoup as Soup -from click.testing import CliRunner -from jinja2 import ChoiceLoader, FileSystemLoader - -from datasette import cli, hookimpl -from datasette.app import Datasette -from datasette.filters import FilterArguments -from datasette.fixtures import TABLES -from datasette.permissions import Action, PermissionSQL -from datasette.plugins import DEFAULT_PLUGINS, get_plugins, pm -from datasette.resources import DatabaseResource -from datasette.utils import StartupError, await_me_maybe -from datasette.utils.sqlite import sqlite3 - -from .fixtures import ( - PLUGINS_DIR, - TEMP_PLUGIN_SECRET_FILE, - make_app_client, -) -from .fixtures import ( - TestClient as _TestClient, -) +import urllib at_memory_re = re.compile(r" at 0x\w+") @@ -40,7 +35,7 @@ at_memory_re = re.compile(r" at 0x\w+") ) def test_plugin_hooks_have_tests(plugin_hook): """Every plugin hook should be referenced in this test module""" - tests_in_this_module = [t for t in globals() if t.startswith("test_hook_")] + tests_in_this_module = [t for t in globals().keys() if t.startswith("test_hook_")] ok = False for test in tests_in_this_module: if plugin_hook in test: @@ -53,13 +48,6 @@ def test_hook_jump_items_sql(): assert "jump_items_sql" in dir(pm.hook) -def test_hook_shutdown(): - # Detailed behavior (ordering against background-task cancellation and - # close(), idempotency, exception handling, sync vs async support) is - # covered in tests/test_shutdown.py. - assert "shutdown" in dir(pm.hook) - - @pytest.mark.asyncio async def test_hook_plugins_dir_plugin_prepare_connection(ds_client): response = await ds_client.get( @@ -137,11 +125,11 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): response = await ds_client.get(path) assert response.status_code == 200 links = Soup(response.text, "html.parser").find_all("link") - special_href = next( + special_href = [ link for link in links if link.attrs["href"].endswith("/extra-css-urls-demo.css") - )["href"] + ][0]["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] actual_decoded_object = json.loads(base64.b64decode(encoded).decode("utf8")) @@ -164,7 +152,7 @@ async def test_hook_extra_js_urls(ds_client): "type": "module", }, ]: - assert any(s == attrs for s in script_attrs), f"Expected: {attrs}" + assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs) @pytest.mark.asyncio @@ -327,8 +315,7 @@ async def test_plugin_config_env_from_list(ds_client): @pytest.mark.asyncio async def test_plugin_config_file(ds_client): - # Blocking write is fine here - it is tiny test setup, not request handling - with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: # noqa: ASYNC230 + with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -429,72 +416,6 @@ def test_hook_extra_template_vars(restore_working_directory): } == extra_template_vars_from_awaitable -@pytest.mark.asyncio -@pytest.mark.parametrize( - "return_style", ["direct", "callable", "async_callable", "awaitable"] -) -async def test_hook_extra_template_vars_none(ds_client, return_style): - class OtherPlugin: - @hookimpl - def extra_template_vars(self): - return {"other": "present"} - - class ConditionalPlugin: - @hookimpl - def extra_template_vars(self, view_name): - def inner(): - if view_name == "database": - return {"conditional": "database"} - - async def async_inner(): - return inner() - - if return_style == "direct": - return inner() - elif return_style == "callable": - return inner - elif return_style == "async_callable": - return async_inner - else: - return async_inner() - - other_plugin = OtherPlugin() - conditional_plugin = ConditionalPlugin() - pm.register(other_plugin) - pm.register(conditional_plugin) - try: - template = ds_client.ds.get_jinja_environment().from_string( - "{{ other }}:{{ conditional|default('missing') }}" - ) - for view_name, expected in ( - ("database", "present:database"), - ("index", "present:missing"), - ): - rendered = await ds_client.ds.render_template(template, view_name=view_name) - assert rendered == expected - finally: - pm.unregister(conditional_plugin) - pm.unregister(other_plugin) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("invalid_value", [False, 0, "", [], ()]) -async def test_hook_extra_template_vars_invalid(ds_client, invalid_value): - class InvalidPlugin: - @hookimpl - def extra_template_vars(self): - return lambda: invalid_value - - plugin = InvalidPlugin() - pm.register(plugin) - try: - template = ds_client.ds.get_jinja_environment().from_string("test") - with pytest.raises(AssertionError, match="extra_vars is of type"): - await ds_client.ds.render_template(template) - finally: - pm.unregister(plugin) - - def test_plugins_async_template_function(restore_working_directory): with make_app_client( template_dir=str(pathlib.Path(__file__).parent / "test_templates") @@ -902,7 +823,7 @@ def test_hook_register_routes_with_datasette(configured_path): assert response.status_code == 200 assert configured_path.upper() == response.text # Other one should 404 - other_path = next(p for p in ("path1", "path2") if configured_path != p) + other_path = [p for p in ("path1", "path2") if configured_path != p][0] assert client.get(f"/{other_path}/", follow_redirects=True).status_code == 404 @@ -1007,7 +928,7 @@ async def test_plugin_startup_can_add_queries(): await datasette.add_query( "data", "from_startup", - f"select {result.first()[0]}", + "select {}".format(result.first()[0]), source="plugin", ) @@ -1119,7 +1040,7 @@ async def test_hook_handle_exception(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async")) async def test_hook_handle_exception_custom_response(ds_client, param): - response = await ds_client.get(f"/trigger-error?{param}=1") + response = await ds_client.get("/trigger-error?{}=1".format(param)) assert response.text == param @@ -1373,8 +1294,7 @@ async def test_hook_filters_from_request(ds_client): ds_client.ds.pm.register(ReturnNothingPlugin(), name="ReturnNothingPlugin") response = await ds_client.get("/fixtures/facetable?_nothing=1") - summary = Soup(response.text, "html.parser").select_one(".table-summary") - assert summary.get_text(" ", strip=True) == "0 rows where NOTHING" + assert "0 rows\n where NOTHING" in response.text json_response = await ds_client.get("/fixtures/facetable.json?_nothing=1") assert json_response.json()["rows"] == [] ds_client.ds.pm.unregister(name="ReturnNothingPlugin") @@ -1454,7 +1374,7 @@ async def test_hook_register_actions_no_duplicates(duplicate): # This should error: with pytest.raises(StartupError) as ex: await ds.invoke_startup() - assert f"Duplicate action {duplicate}" in str(ex.value) + assert "Duplicate action {}".format(duplicate) in str(ex.value) @pytest.mark.asyncio @@ -1562,7 +1482,7 @@ async def test_plugin_is_installed(): datasette.pm.register(DummyPlugin(), name="DummyPlugin") response = await datasette.client.get("/-/plugins.json") assert response.status_code == 200 - installed_plugins = {p["name"] for p in response.json()} + installed_plugins = {p["name"] for p in response.json()["plugins"]} assert "DummyPlugin" in installed_plugins finally: diff --git a/tests/test_pr76_fts_policy.py b/tests/test_pr76_fts_policy.py deleted file mode 100644 index 2d4086b3..00000000 --- a/tests/test_pr76_fts_policy.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Policy and compatibility coverage for PR #76, run against the fixed checkout.""" - -import uuid - -import pytest - -from datasette.app import Datasette -from datasette.resources import TableResource -from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies - - -@pytest.mark.parametrize("vocab_name", ["words", "name USING fts4aux", 'quoted"name']) -@pytest.mark.parametrize( - "module,arguments", - [ - ("fts5vocab", "'Search,Index', 'row'"), - ("fts5vocab", "'SEARCH,INDEX', 'col'"), - ("fts5vocab", "'Search,Index', 'instance'"), - ("fts4aux", "'Search,Index'"), - ], -) -def test_vocabulary_dependency_identity(module, arguments, vocab_name): - conn = sqlite3.connect(":memory:") - try: - fts = "fts5" if module == "fts5vocab" else "fts4" - conn.execute(f'create virtual table "Search,Index" using {fts}(body)') - quoted_name = '"' + vocab_name.replace('"', '""') + '"' - conn.execute( - f"create virtual table {quoted_name} USING /* module */ {module}({arguments})" - ) - assert sqlite_derived_table_dependencies(conn)[vocab_name] == "Search,Index" - finally: - conn.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("module", ["fts5", "fts4"]) -@pytest.mark.parametrize("external_content", [False, True], ids=["one-hop", "two-hop"]) -@pytest.mark.parametrize( - "source_allowed,vocab_allowed", [(False, True), (True, False), (True, True)] -) -async def test_vocabulary_immediate_source_permissions( - module, external_content, source_allowed, vocab_allowed -): - ds = Datasette( - memory=True, - config={ - "databases": { - "data": { - "tables": { - "search": { - "permissions": { - "view-table": ( - {"id": "reader"} if source_allowed else False - ) - } - }, - "words": {"permissions": {"view-table": vocab_allowed}}, - } - } - } - }, - ) - db = ds.add_memory_database(uuid.uuid4().hex, name="data") - await db.execute_write("create table documents(body text)") - options = "body, content='documents'" if external_content else "body" - await db.execute_write(f"create virtual table search using {module}({options})") - definition = ( - "fts5vocab('SEARCH', 'row')" if module == "fts5" else "fts4aux('SEARCH')" - ) - await db.execute_write(f"create virtual table words using {definition}") - await ds.invoke_startup() - try: - actor = {"id": "reader"} - expected = source_allowed and vocab_allowed and not external_content - for name in ("words", "WORDS"): - assert ( - await ds.allowed( - action="view-table", - resource=TableResource("data", name), - actor=actor, - ) - is expected - ) - resources = await ds.allowed_resources( - "view-table", parent="data", actor=actor, include_is_private=True - ) - words = [r for r in resources.resources if r.child == "words"] - assert bool(words) is expected - if expected: - assert words[0].private - assert not await ds.allowed( - action="view-table", resource=TableResource("data", "words") - ) - # Dropping the source invalidates dependency metadata and remains denied. - await db.execute_write("drop table search") - assert not await ds.allowed( - action="view-table", resource=TableResource("data", "words"), actor=actor - ) - finally: - ds.close() - - -@pytest.mark.parametrize( - "module,definition", - [ - ("fts5", "fts5vocab('main', 'search', 'row')"), - ("fts4", "fts4aux('main', 'search')"), - ], -) -def test_cross_schema_vocabulary_is_unresolved(module, definition): - conn = sqlite3.connect(":memory:") - try: - conn.execute(f"create virtual table search using {module}(body)") - conn.execute(f"create virtual table temp.words using {definition}") - # Cross-schema ownership is not representable by the current map. - # The source is itself derived, so the immediate-source policy denies it. - assert ( - sqlite_derived_table_dependencies(conn, schema="temp")["words"] == "words" - ) - finally: - conn.close() - - -@pytest.mark.parametrize( - "definition", - [ - """CREATE VIRTUAL TABLE"words"USING"fts5vocab"('search', 'row')""", - """CREATE VIRTUAL TABLE[words]USING[fts5vocab]('search', 'row')""", - """CREATE VIRTUAL TABLE`words`USING`fts5vocab`('search', 'row')""", - ], -) -def test_vocabulary_quoted_token_boundaries(definition): - conn = sqlite3.connect(":memory:") - try: - conn.execute("create virtual table search using fts5(body)") - conn.execute(definition) - assert sqlite_derived_table_dependencies(conn)["words"] == "search" - finally: - conn.close() diff --git a/tests/test_pr76_statistics_policy.py b/tests/test_pr76_statistics_policy.py deleted file mode 100644 index 6ebf10b9..00000000 --- a/tests/test_pr76_statistics_policy.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Statistics access policy and plugin replacement coverage for PR #76.""" - -import uuid - -import pytest - -from datasette import hookimpl -from datasette.app import Datasette -from datasette.permissions import PermissionSQL -from datasette.resources import TableResource - - -@pytest.mark.asyncio -@pytest.mark.parametrize("scope", [None, "global", "database", "table", "root"]) -async def test_statistics_denied_despite_allow_rules(scope): - config = {"databases": {"data": {"tables": {"sqlite_stat1": {}}}}} - grant = {"view-table": True} - if scope == "global": - config["permissions"] = grant - elif scope == "database": - config["databases"]["data"]["permissions"] = grant - elif scope == "table": - config["databases"]["data"]["tables"]["sqlite_stat1"]["permissions"] = grant - ds = Datasette(memory=True, config=config) - ds.root_enabled = scope == "root" - actor = {"id": "root"} if scope == "root" else {"id": "reader"} - db = ds.add_memory_database(uuid.uuid4().hex, name="data") - await db.execute_write("create table items(value text)") - await db.execute_write("create index items_value on items(value)") - await db.execute_write("insert into items values ('example')") - await db.execute_write("analyze") - await ds.invoke_startup() - try: - assert "view-sqlite-statistics" not in ds.actions - for name in ("sqlite_stat1", "SQLITE_STAT1"): - assert not await ds.allowed( - action="view-table", resource=TableResource("data", name), actor=actor - ) - for suffix in ("", ".json", ".csv"): - assert ( - await ds.client.get(f"/data/sqlite_stat1{suffix}", actor=actor) - ).status_code == 403 - resources = await ds.allowed_resources("view-table", parent="data", actor=actor) - assert "sqlite_stat1" not in {r.child for r in resources.resources} - assert "items" in {r.child for r in resources.resources} - finally: - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "table", ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] -) -@pytest.mark.parametrize("default_deny", [False, True]) -async def test_statistics_names_denied(table, default_deny): - ds = Datasette(memory=True, default_deny=default_deny) - ds.root_enabled = True - await ds.invoke_startup() - try: - for name in (table, table.upper()): - assert not await ds.allowed( - action="view-table", - resource=TableResource("_memory", name), - actor={"id": "root"}, - ) - finally: - ds.close() - - -@pytest.mark.asyncio -async def test_plugin_can_replace_statistics_policy(): - class ReplacementPolicy: - @hookimpl - def permission_resources_sql(self, action, actor): - if action == "view-table": - return PermissionSQL( - sql="SELECT 'data' AS parent, 'sqlite_stat1' AS child, :statistics_allowed AS allow, 'custom statistics policy' AS reason", - params={"statistics_allowed": int(actor == {"id": "reader"})}, - ) - - ds = Datasette(memory=True) - db = ds.add_memory_database(uuid.uuid4().hex, name="data") - await db.execute_write("create table items(value text)") - await db.execute_write("analyze") - await ds.invoke_startup() - name = "datasette.default_permissions.sqlite_statistics" - original = ds.pm.unregister(name=name) - assert original is not None - replacement = ReplacementPolicy() - ds.pm.register(replacement, name="test-replacement-statistics-policy") - try: - actor = {"id": "reader"} - assert await ds.allowed( - action="view-table", - resource=TableResource("data", "sqlite_stat1"), - actor=actor, - ) - assert not await ds.allowed( - action="view-table", resource=TableResource("data", "sqlite_stat1") - ) - resources = await ds.allowed_resources( - "view-table", parent="data", actor=actor, include_is_private=True - ) - stats = [r for r in resources.resources if r.child == "sqlite_stat1"] - assert len(stats) == 1 and stats[0].private - assert ( - await ds.client.get("/data/sqlite_stat1.json", actor=actor) - ).status_code == 200 - assert (await ds.client.get("/data/sqlite_stat1.json")).status_code == 403 - finally: - ds.pm.unregister(replacement) - ds.pm.register(original, name=name) - ds.close() diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index aebcaa33..6617bc77 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -1,12 +1,10 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import json import os -import textwrap -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli +import textwrap @pytest.mark.serial @@ -72,7 +70,9 @@ def test_publish_cloudrun_prompts_for_service( ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} input-service --max-instances 1", + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format( + tag + ), shell=True, ), ] @@ -107,7 +107,9 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} test --max-instances 1", + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format( + tag + ), shell=True, ), ] @@ -184,13 +186,13 @@ def test_publish_cloudrun_memory_cpu( tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test" expected_call = ( "gcloud run deploy --allow-unauthenticated --platform=managed" - f" --image {tag} test" + " --image {} test".format(tag) ) expected_build_call = f"gcloud builds submit --tag {tag}" if memory: - expected_call += f" --memory {memory}" + expected_call += " --memory {}".format(memory) if cpu: - expected_call += f" --cpu {cpu}" + expected_call += " --cpu {}".format(cpu) if timeout: expected_build_call += f" --timeout {timeout}" # max_instances defaults to 1 diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 4302ed94..cab83654 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,11 +1,9 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import os import pathlib -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli @pytest.mark.serial diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py index 9b17d24b..3af1aace 100644 --- a/tests/test_pytest_autoclose_plugin.py +++ b/tests/test_pytest_autoclose_plugin.py @@ -20,7 +20,6 @@ def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess: cwd=str(tmp_path), capture_output=True, text=True, - check=False, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index ebe8b832..ffa948a9 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -15,29 +15,37 @@ from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" ) -EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" +EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "\n".join( + ( + "create table new_table (", + " id integer primary key,", + " name text", + " -- created text default (datetime('now'))", + ")", + ) +) def _template_option_attributes(html, table): - match = re.search(rf'      ', + ''.format( + i, i + ), f'', f'', f'', @@ -1710,7 +1667,7 @@ async def test_row_update_sets_message(): assert response.status_code == 200 assert response.json()["rows"][0]["name"] == long_name assert ds.unsign(response.cookies["ds_messages"], "messages") == [ - [f"Updated row 1 ({truncated_name})", ds.INFO] + ["Updated row 1 ({})".format(truncated_name), ds.INFO] ] finally: ds.close() @@ -1723,9 +1680,9 @@ def test_table_data_uses_base_url(app_client_base_url_prefix): import re soup = Soup(response.text, "html.parser") - table_script = next( + table_script = [ s for s in soup.find_all("script") if "_datasetteTableData" in (s.string or "") - ) + ][0] match = re.search( r"window\._datasetteTableData\s*=\s*({.*?});", table_script.string, @@ -1753,9 +1710,8 @@ def test_table_fragment_custom_table_include(): @pytest.mark.asyncio async def test_table_fragment_uses_render_cell_hook(): - from markupsafe import Markup - from datasette import hookimpl + from markupsafe import Markup class TestRenderCellPlugin: __name__ = "TestRenderCellPlugin" @@ -1763,7 +1719,7 @@ async def test_table_fragment_uses_render_cell_hook(): @hookimpl def render_cell(self, value, column, table, database): if database == "data" and table == "items" and column == "name": - return Markup(f"{value}") + return Markup("{}".format(value)) return None ds = Datasette(memory=True) @@ -2302,16 +2258,18 @@ def test_allow_facet_off(allow_facet): ) async def test_format_of_binary_links(size, title, length_bytes): ds = Datasette() - db_name = f"binary-links-{size}" + db_name = "binary-links-{}".format(size) db = ds.add_memory_database(db_name) - sql = f"select zeroblob({size}) as blob" - await db.execute_write(f"create table blobs as {sql}") - response = await ds.client.get(f"/{db_name}/blobs") + sql = "select zeroblob({}) as blob".format(size) + await db.execute_write("create table blobs as {}".format(sql)) + response = await ds.client.get("/{}/blobs".format(db_name)) assert response.status_code == 200 - expected = f"{title}><Binary: {length_bytes} bytes>" + expected = "{}><Binary: {} bytes>".format(title, length_bytes) assert expected in response.text # And test with arbitrary SQL query too - sql_response = await ds.client.get(f"{db_name}/-/query", params={"sql": sql}) + sql_response = await ds.client.get( + "{}/-/query".format(db_name), params={"sql": sql} + ) assert sql_response.status_code == 200 assert expected in sql_response.text diff --git a/tests/test_table_resource_identity.py b/tests/test_table_resource_identity.py deleted file mode 100644 index 35b4dcc6..00000000 --- a/tests/test_table_resource_identity.py +++ /dev/null @@ -1,420 +0,0 @@ -"""Table permission identities must agree with SQLite identifier resolution.""" - -import uuid -from unittest.mock import AsyncMock - -import pytest - -from datasette import hookimpl -from datasette.app import Datasette -from datasette.default_permissions import restrictions_allow_action -from datasette.permissions import Action, PermissionSQL, _permission_check_cache -from datasette.resources import QueryResource, TableResource -from datasette.utils.actions_sql import explain_permission_for_resource -from datasette.utils.asgi import Forbidden -from datasette.utils.permissions import gather_permission_sql_from_hooks - - -@pytest.mark.asyncio -@pytest.mark.parametrize("kind", ["table", "view"]) -@pytest.mark.parametrize("spelling", ["Inventory", "inventory", "INVENTORY"]) -@pytest.mark.parametrize("allowed", [False, True]) -@pytest.mark.parametrize("rule_spelling", ["Inventory", "iNvEnToRy"]) -async def test_table_permission_identity( - kind, spelling, allowed, rule_spelling, monkeypatch -): - ds = Datasette( - config={ - "permissions": {"view-table": not allowed, "insert-row": not allowed}, - "databases": { - "data": { - "tables": { - rule_spelling: { - "permissions": { - "view-table": allowed, - "insert-row": allowed, - } - } - } - } - }, - } - ) - db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - cache_token = _permission_check_cache.set({}) - try: - await db.execute_write( - "create table Inventory (id integer primary key)" - if kind == "table" - else "create view Inventory as select 1 as id" - ) - await ds.invoke_startup() - # Identity matching needs no target-schema lookup. Derived-table - # permissions may still check the schema version. All spellings and - # API entry points should share the existing permission result cache. - target_execute = AsyncMock(wraps=db.execute) - monkeypatch.setattr(db, "execute", target_execute) - internal_execute = AsyncMock(wraps=ds.get_internal_database().execute) - monkeypatch.setattr(ds.get_internal_database(), "execute", internal_execute) - resource = TableResource("data", spelling) - assert await ds.allowed_many( - actions=["view-table", "insert-row"], resource=resource - ) == {"view-table": allowed, "insert-row": allowed} - assert await ds.allowed(action="view-table", resource=resource) is allowed - assert await ds.check_visibility(None, "view-table", resource) == ( - allowed, - False, - ) - if allowed: - await ds.ensure_permission(action="view-table", resource=resource) - else: - with pytest.raises(Forbidden): - await ds.ensure_permission(action="view-table", resource=resource) - assert resource.child == spelling # Do not mutate caller-owned resources. - for variant in ("Inventory", "inventory", "INVENTORY"): - assert ( - await ds.allowed( - action="view-table", resource=TableResource("data", variant) - ) - is allowed - ) - assert internal_execute.await_count == 1 - assert all( - call.args[0] == "PRAGMA schema_version" - for call in target_execute.await_args_list - ) - assert all(key[3] == "inventory" for key in _permission_check_cache.get()) - finally: - _permission_check_cache.reset(cache_token) - ds.close() - - -@pytest.mark.asyncio -async def test_other_permission_identities_are_preserved(): - ds = Datasette( - config={ - "databases": { - "data": { - "tables": { - "Äpfel": {"permissions": {"view-table": False}}, - "Future": {"permissions": {"view-table": False}}, - }, - "queries": { - "Report": { - "sql": "select 1", - "permissions": {"view-query": False}, - }, - "report": { - "sql": "select 1", - "permissions": {"view-query": True}, - }, - }, - } - } - } - ) - db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - try: - await db.execute_write('create table "Äpfel" (id integer primary key)') - await db.execute_write('create table "äpfel" (id integer primary key)') - await db.execute_write("create table Report (id integer primary key)") - await ds.invoke_startup() - # SQLite folds ASCII identifier casing, not Unicode casing. - for name, expected in [ - ("ÄPFEL", False), - ("äPFEL", True), - ("Future", False), - ("future", False), - ]: - assert ( - await ds.allowed( - action="view-table", resource=TableResource("data", name) - ) - is expected - ) - # Query names remain case-sensitive even when a table has the same name. - for name, expected in [("Report", False), ("report", True)]: - assert ( - await ds.allowed( - action="view-query", resource=QueryResource("data", name) - ) - is expected - ) - finally: - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("allow", [True, False, {"id": "reader"}]) -async def test_table_listings_and_explanations(allow): - ds = Datasette( - config={ - "databases": { - "data": { - "tables": { - "inventory": {"permissions": {"view-table": allow}}, - } - } - } - } - ) - db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - try: - await db.execute_write("create table Inventory (id integer primary key)") - await db.execute_write("create view InventoryView as select id from Inventory") - await ds.invoke_startup() - for actor in (None, {"id": "reader"}): - expected = allow is True or (isinstance(allow, dict) and actor == allow) - explanation = await explain_permission_for_resource( - datasette=ds, - actor=actor, - action="view-table", - parent="data", - child="INVENTORY", - ) - assert explanation["allowed"] is expected - assert explanation["winning_scope"] == "resource" - assert any( - "data/inventory" in rule["reason"] - for rule in explanation["matched_rules"] - ) - page = await ds.allowed_resources( - "view-table", - actor, - parent="data", - include_is_private=True, - include_reasons=True, - limit=1, - ) - resources = [resource async for resource in page.all()] - matching = [r for r in resources if r.child == "Inventory"] - assert bool(matching) is expected - assert len(matching) <= 1 - if matching: - assert matching[0].private is isinstance(allow, dict) - assert any(r.child == "InventoryView" for r in resources) - finally: - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("deny_first", [True, False]) -async def test_case_variant_rules_deny_wins(deny_first): - rules = [("inventory", False), ("INVENTORY", True)] - if not deny_first: - rules.reverse() - ds = Datasette( - config={ - "databases": { - "data": { - "tables": { - name: {"permissions": {"view-table": allow}} - for name, allow in rules - } - } - } - } - ) - db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - try: - await db.execute_write("create table Inventory (id integer primary key)") - await ds.invoke_startup() - assert not await ds.allowed( - action="view-table", resource=TableResource("data", "Inventory") - ) - assert not ( - await ds.allowed_resources( - "view-table", parent="data", include_is_private=True - ) - ).resources - explanation = await explain_permission_for_resource( - datasette=ds, - actor=None, - action="view-table", - parent="data", - child="Inventory", - ) - assert not explanation["allowed"] - assert any( - rule["effect"] == "allow" and not rule["decisive"] - for rule in explanation["matched_rules"] - ) - assert any( - rule["effect"] == "deny" and rule["decisive"] - for rule in explanation["matched_rules"] - ) - finally: - ds.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("config_style", ["allow", "permissions"]) -@pytest.mark.parametrize("allowed", [True, False]) -async def test_case_variant_token_restrictions(config_style, allowed): - table_config = ( - {"allow": allowed} - if config_style == "allow" - else {"permissions": {"view-table": allowed}} - ) - ds = Datasette( - config={"databases": {"data": {"tables": {"Inventory": table_config}}}} - ) - db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - actor = {"id": "reader", "_r": {"r": {"data": {"inventory": ["vt"]}}}} - try: - await db.execute_write("create table Inventory (id integer primary key)") - await ds.invoke_startup() - assert restrictions_allow_action( - ds, actor["_r"], "view-table", ("data", "INVENTORY") - ) - assert not restrictions_allow_action( - ds, actor["_r"], "view-table", ("Data", "Inventory") - ) - assert ( - await ds.allowed( - action="view-table", - resource=TableResource("data", "INVENTORY"), - actor=actor, - ) - is allowed - ) - page = await ds.allowed_resources("view-table", actor, parent="data") - assert [(r.parent, r.child) for r in page.resources] == ( - [("data", "Inventory")] if allowed else [] - ) - explanation = await explain_permission_for_resource( - datasette=ds, - actor=actor, - action="view-table", - parent="data", - child="Inventory", - ) - assert explanation["restriction_allowed"] - assert explanation["allowed"] is allowed - finally: - ds.close() - - -@pytest.mark.asyncio -async def test_plugin_restriction_intersection_and_dependencies(): - class Plugin: - @hookimpl - def register_actions(self, datasette): - return [ - Action( - name="inspect-inventory", - description="Inspect inventory", - resource_class=TableResource, - also_requires="view-table", - ) - ] - - @hookimpl - def permission_resources_sql(self, action): - if action not in ("view-table", "inspect-inventory"): - return None - return [ - PermissionSQL( - sql="SELECT 'data' AS parent, 'INVENTORY' AS child, 1 AS allow, 'inventory grant' AS reason", - restriction_sql="SELECT 'data' AS parent, 'inventory' AS child", - ), - PermissionSQL( - restriction_sql="SELECT 'data' AS parent, 'InVeNtOrY' AS child" - ), - ] - - ds = Datasette(default_deny=True) - ds.pm.register(Plugin(), name="identity-test") - db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - try: - await db.execute_write("create table Inventory (id integer primary key)") - await db.execute_write("create table Other (id integer primary key)") - await ds.invoke_startup() - for action in ("view-table", "inspect-inventory"): - assert await ds.allowed( - action=action, resource=TableResource("data", "Inventory") - ) - assert not await ds.allowed( - action=action, resource=TableResource("data", "Other") - ) - resources = ( - await ds.allowed_resources( - action, parent="data", include_is_private=True - ) - ).resources - assert [r.child for r in resources] == ["Inventory"] - explanation = await explain_permission_for_resource( - datasette=ds, - actor=None, - action=action, - parent="data", - child="Inventory", - ) - assert explanation["allowed"] - assert all(item["allowed"] for item in explanation["restrictions"]) - finally: - ds.pm.unregister(name="identity-test") - ds.close() - - -@pytest.mark.asyncio -async def test_shared_plugin_rule_keeps_query_identity_and_original_sql(): - shared = PermissionSQL( - sql="SELECT 'data' AS parent, 'Inventory' AS child, 0 AS allow, 'shared deny' AS reason" - ) - original_sql = shared.sql - - class Plugin: - @hookimpl - def permission_resources_sql(self, action): - if action in ("view-table", "view-query"): - return shared - - ds = Datasette( - config={ - "databases": { - "data": { - "queries": { - "Inventory": "select 1", - "inventory": "select 1", - } - } - } - } - ) - ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data") - ds.pm.register(Plugin(), name="identity-test") - try: - await ds.invoke_startup() - for _ in range(2): - await gather_permission_sql_from_hooks( - datasette=ds, actor=None, action="view-table" - ) - assert shared.sql == original_sql - assert not await ds.allowed( - action="view-table", resource=TableResource("data", "inventory") - ) - assert await ds.allowed( - action="view-query", resource=QueryResource("data", "inventory") - ) - assert not await ds.allowed( - action="view-query", resource=QueryResource("data", "Inventory") - ) - assert await ds.allowed( - action="view-table", resource=TableResource("Data", "Inventory") - ) - assert restrictions_allow_action( - ds, - {"r": {"data": {"Inventory": ["vq"]}}}, - "view-query", - ("data", "Inventory"), - ) - assert not restrictions_allow_action( - ds, - {"r": {"data": {"Inventory": ["vq"]}}}, - "view-query", - ("data", "inventory"), - ) - finally: - ds.pm.unregister(name="identity-test") - ds.close() diff --git a/tests/test_tasks_endpoint.py b/tests/test_tasks_endpoint.py deleted file mode 100644 index be174dbe..00000000 --- a/tests/test_tasks_endpoint.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Tests for the /-/tasks introspection endpoint. - -/-/tasks exposes datasette._background_tasks (see tests/test_background_tasks.py -for the supervisor machinery itself) the same way /-/threads exposes threading -internals: gated behind the permissions-debug permission, JSON-only. -""" - -import asyncio -import contextlib -import functools - -import pytest - -from datasette.app import Datasette - - -async def example_task(datasette): - pass - - -class ExampleWorker: - async def run(self, datasette): - pass - - async def __call__(self, datasette): - pass - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "func, qualified_name", - [ - (example_task, "example_task"), - (functools.partial(example_task), "example_task"), - (ExampleWorker().run, "ExampleWorker.run"), - (ExampleWorker(), "ExampleWorker.__call__"), - ], -) -async def test_task_function_path(func, qualified_name): - ds = Datasette(memory=True) - ds.root_enabled = True - handle = ds.add_background_task(func, name="custom-name") - try: - response = await ds.client.get("/-/tasks.json", actor={"id": "root"}) - assert response.status_code == 200 - task = response.json()["tasks"][0] - assert task["name"] == "custom-name" - assert task["function"] == f"{__name__}.{qualified_name}" - assert handle.function == task["function"] - assert "plugin" not in task - await handle.task - html = await ds.client.get("/-/tasks", actor={"id": "root"}) - assert html.status_code == 200 - assert task["function"] in html.text - finally: - await ds.invoke_shutdown() - - -@pytest.mark.asyncio -async def test_tasks_requires_permissions_debug(): - ds = Datasette(memory=True) - ds.root_enabled = True - - denied = await ds.client.get("/-/tasks.json") - assert denied.status_code == 403 - - allowed = await ds.client.get("/-/tasks.json", actor={"id": "root"}) - assert allowed.status_code == 200 - data = allowed.json() - assert data["ok"] is True - assert "tasks" in data - assert "launched" in data - - -@pytest.mark.asyncio -async def test_running_and_crashed_task_states(): - ds = Datasette(memory=True) - ds.root_enabled = True - - async def long_running(datasette): - await asyncio.Event().wait() - - async def crashing_task(datasette): - raise RuntimeError("kaboom") - - long_handle = ds.add_background_task(long_running, name="long-runner") - crash_handle = ds.add_background_task(crashing_task, name="crashing_task") - - await ds.start_background_tasks() - - # Let the crashing_task run to completion and its done-callback (which sets - # handle.state = "crashed") actually fire before we read state back out. - await asyncio.wait_for( - asyncio.gather(crash_handle.task, return_exceptions=True), timeout=5 - ) - await asyncio.sleep(0) - - try: - response = await ds.client.get("/-/tasks.json", actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["launched"] is True - - by_name = {t["name"]: t for t in data["tasks"]} - assert by_name["long-runner"]["state"] == "running" - assert by_name["long-runner"]["exception"] is None - assert by_name["long-runner"]["started_at"] is not None - - crashed = by_name["crashing_task"] - assert crashed["function"] == ( - f"{__name__}.test_running_and_crashed_task_states..crashing_task" - ) - assert crashed["state"] == "crashed" - assert crashed["exception"] is not None - assert isinstance(crashed["exception"], str) - assert "kaboom" in crashed["exception"] - assert "RuntimeError" in crashed["exception"] - finally: - long_handle.cancel() - with contextlib.suppress(asyncio.CancelledError): - await long_handle.task diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 691c2d64..7923d0e7 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field import pytest -from datasette.app import TEMPLATE_BASE_CONTEXT, Datasette +from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database from datasette.template_contexts import PAGES, documented_context_keys @@ -40,17 +40,17 @@ def test_documented_fields(): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert ( - context_field.help - ), f"{klass.__name__}.{context_field.name} is missing documentation" + assert context_field.help, "{}.{} is missing documentation".format( + klass.__name__, context_field.name + ) @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_has_docstring_and_documented_template(klass): - assert klass.__doc__, f"{klass.__name__} is missing a docstring" - assert ( - klass.documented_template - ), f"{klass.__name__} is missing a documented_template" + assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) + assert klass.documented_template, "{} is missing a documented_template".format( + klass.__name__ + ) def test_from_extra_documentation_comes_from_the_extra_class(): @@ -105,7 +105,7 @@ def isolate_extra_template_vars_plugins(): # for the rest of the process. The contract documents plugin-free # Datasette core, so unregister any non-default plugin that adds # template variables via the extra_template_vars hook - from datasette.plugins import DEFAULT_PLUGINS, pm + from datasette.plugins import pm, DEFAULT_PLUGINS hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} removed = [] @@ -182,18 +182,18 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - f"Undocumented keys in {page_name} template context: {sorted(undocumented)} - add them to the " - "page's Context class" + "Undocumented keys in {} template context: {} - add them to the " + "page's Context class".format(page_name, sorted(undocumented)) ) assert not no_longer_present, ( - f"Documented keys missing from {page_name} template context: {sorted(no_longer_present)} - this would " - "break custom templates" + "Documented keys missing from {} template context: {} - this would " + "break custom templates".format(page_name, sorted(no_longer_present)) ) def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): - assert doc, f"Base context key {name} is missing docs" + assert doc, "Base context key {} is missing docs".format(name) def test_template_context_docs_cover_every_documented_key(): @@ -201,14 +201,15 @@ def test_template_context_docs_cover_every_documented_key(): assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() for name in TEMPLATE_BASE_CONTEXT: - assert f"``{name}``" in docs, name + assert "``{}``".format(name) in docs, name for page_name, klass in PAGES.items(): title = "{} page".format(klass.__name__.removesuffix("Context")) assert title in docs, title for context_field in klass.documented_fields(): + assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( + context_field.name, page_name + ) assert ( - f"``{context_field.name}``" in docs - ), f"{context_field.name} ({page_name} page)" - assert ( - f"``{context_field.name}`` - ``{context_field.type_name}``" in docs - ), f"{context_field.name} type ({page_name} page)" + "``{}`` - ``{}``".format(context_field.name, context_field.type_name) + in docs + ), "{} type ({} page)".format(context_field.name, page_name) diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 10021ddf..f5bbfead 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -2,17 +2,16 @@ Tests for the register_token_handler plugin hook. """ -import pytest - from datasette.app import Datasette from datasette.hookspecs import hookimpl from datasette.plugins import pm from datasette.tokens import ( - SignedTokenHandler, TokenHandler, TokenInvalid, TokenRestrictions, + SignedTokenHandler, ) +import pytest @pytest.fixture diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 21cfa952..9db211d3 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,5 +1,4 @@ import pytest - from .fixtures import make_app_client @@ -76,9 +75,10 @@ async def test_trace_child_tasks_resets_contextvar_on_exception(): from datasette import tracer before = tracer.trace_task_id.get() - with pytest.raises(ValueError), tracer.trace_child_tasks(): - assert tracer.trace_task_id.get() is not None - raise ValueError("simulated error") + with pytest.raises(ValueError): + with tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") # The contextvar must be reset even though the block raised assert tracer.trace_task_id.get() == before diff --git a/tests/test_utils.py b/tests/test_utils.py index 68d4a504..a535ca93 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,25 +2,22 @@ Tests for various datasette helper functions. """ -import hashlib -import json -import os -import pathlib -import tempfile -from unittest.mock import patch - -import pytest - -from datasette import utils from datasette.app import Datasette +from datasette import utils from datasette.utils.asgi import Request from datasette.utils.sqlite import ( sqlite3, - sqlite_derived_table_dependencies, sqlite_hidden_table_names, sqlite_table_type, supports_returning, ) +import hashlib +import json +import os +import pathlib +import pytest +import tempfile +from unittest.mock import patch @pytest.mark.parametrize( @@ -197,7 +194,7 @@ def test_validate_sql_select_good(good_sql): @pytest.mark.parametrize("open_quote,close_quote", [('"', '"'), ("[", "]")]) def test_detect_fts(open_quote, close_quote): - sql = f""" + sql = """ CREATE TABLE "Dumb_Table" ( "TreeID" INTEGER, "qSpecies" TEXT @@ -212,9 +209,9 @@ def test_detect_fts(open_quote, close_quote): "qCaretaker" TEXT ); CREATE VIEW Test_View AS SELECT * FROM Dumb_Table; - CREATE VIRTUAL TABLE {open_quote}Street_Tree_List_fts{close_quote} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open_quote}Street_Tree_List{close_quote}); + CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open}Street_Tree_List{close}); CREATE VIRTUAL TABLE r USING rtree(a, b, c); - """ + """.format(open=open_quote, close=close_quote) conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) assert None is utils.detect_fts(conn, "Dumb_Table") @@ -228,8 +225,6 @@ def test_detect_fts(open_quote, close_quote): "identifier,expected", ( ("plain", "plain"), - ("plain\n", '"plain\n"'), - ("select\n", '"select\n"'), ("select", '"select"'), ("has space", '"has space"'), ("has'quote", '"has\'quote"'), @@ -267,8 +262,8 @@ def test_escape_sqlite_prevents_injection(): conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") malicious = "users] UNION SELECT password FROM users--" - conn.execute(f'CREATE TABLE "{malicious}" (id INTEGER)') - sql = f"select count(*) from {utils.escape_sqlite(malicious)}" + conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) + sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) results = conn.execute(sql).fetchall() conn.close() # The injected UNION must not execute - only the empty malicious table @@ -278,16 +273,16 @@ def test_escape_sqlite_prevents_injection(): @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): - sql = f""" + sql = """ CREATE TABLE [{table}] ( "TreeID" INTEGER, "qSpecies" TEXT ); CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 ("qSpecies", content="{table}"); - """ + """.format(table=table) conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) - assert f"{table}_fts" == utils.detect_fts(conn, table) + assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table) conn.close() @@ -372,46 +367,6 @@ def test_sqlite_hidden_table_names_hides_multiline_content_fts_table(): conn.close() -def test_sqlite_derived_table_dependencies(): - conn = utils.sqlite3.connect(":memory:") - try: - conn.executescript(""" - create table docs(id integer primary key, body text); - create virtual table external_fts5 using fts5( - body, content='docs', content_rowid='id' - ); - create virtual table internal_fts5 using fts5(body); - create virtual table contentless_fts5 using fts5(body, content=''); - create virtual table external_fts4 using fts4(body, content="docs"); - create virtual table internal_fts4 using fts4(body); - create virtual table contentless_fts4 using fts4(body, content=""); - create table [docs, archive](body text); - create virtual table commented_fts5 using fts5( - body, tokenize='porter unicode61', - /* Comments and commas in quoted values must not confuse parsing. */ - content='docs, archive' - ); - create virtual table boxes using rtree(id, minx, maxx, miny, maxy); - """) - - dependencies = sqlite_derived_table_dependencies(conn) - - assert dependencies["external_fts5"] == "docs" - assert dependencies["external_fts4"] == "docs" - assert dependencies["commented_fts5"] == "docs, archive" - assert "contentless_fts5" not in dependencies - assert "contentless_fts4" not in dependencies - assert dependencies["internal_fts5_content"] == "internal_fts5" - assert dependencies["internal_fts4_content"] == "internal_fts4" - assert dependencies["external_fts5_data"] == "external_fts5" - assert dependencies["external_fts4_segments"] == "external_fts4" - assert dependencies["boxes_node"] == "boxes" - assert dependencies["boxes_parent"] == "boxes" - assert dependencies["boxes_rowid"] == "boxes" - finally: - conn.close() - - @pytest.mark.parametrize( "url,expected", [ @@ -735,6 +690,7 @@ def test_resolve_env_secrets(config, expected): [ ({"id": "blah"}, "blah"), ({"id": "blah", "login": "l"}, "l"), + ({"id": "blah", "login": "l"}, "l"), ({"id": "blah", "login": "l", "username": "u"}, "u"), ({"login": "l", "name": "n"}, "n"), ( diff --git a/tests/test_utils_check_callable.py b/tests/test_utils_check_callable.py index 857b73cd..4f72f9ff 100644 --- a/tests/test_utils_check_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,6 +1,5 @@ -import pytest - from datasette.utils.check_callable import check_callable +import pytest class AsyncClass: diff --git a/tests/test_utils_permissions.py b/tests/test_utils_permissions.py index 918dab95..bc3599c2 100644 --- a/tests/test_utils_permissions.py +++ b/tests/test_utils_permissions.py @@ -1,17 +1,14 @@ -from collections.abc import Callable - import pytest - from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.utils.permissions import resolve_permissions_from_catalog +from typing import Callable, List @pytest.fixture def db(): ds = Datasette() import tempfile - from datasette.database import Database path = tempfile.mktemp(suffix="demo.db") @@ -130,7 +127,7 @@ def plugin_root_deny_for_all() -> Callable[[str], PermissionSQL]: def plugin_conflicting_same_child_rules( user: str, parent: str, child: str -) -> list[Callable[[str], PermissionSQL]]: +) -> List[Callable[[str], PermissionSQL]]: def allow_provider(action: str) -> PermissionSQL: return PermissionSQL( """ @@ -280,7 +277,9 @@ async def test_alice_global_allow_with_specific_denies_catalog(db): # Alice can see everything except accounting/sales and hr/* assert "/accounting/sales" in res_denied(rows) for r in rows: - if r["parent"] == "hr" or r["resource"] == "/accounting/sales": + if r["parent"] == "hr": + assert r["allow"] == 0 + elif r["resource"] == "/accounting/sales": assert r["allow"] == 0 else: assert r["allow"] == 1 diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 363814d7..979ff9e1 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -1,7 +1,7 @@ import pytest -from datasette.utils.sql_analysis import analyze_sql_tables from datasette.utils.sqlite import sqlite3 +from datasette.utils.sql_analysis import analyze_sql_tables @pytest.fixture @@ -439,7 +439,7 @@ def test_analyze_attached_database_tables(conn): } -def test_analyze_disables_authorizer_on_error(): +def test_analyze_clears_authorizer_on_error(): class FakeConnection: def __init__(self): self.authorizers = [] @@ -455,5 +455,4 @@ def test_analyze_disables_authorizer_on_error(): with pytest.raises(sqlite3.OperationalError): analyze_sql_tables(conn, "bad SQL") - final_authorizer = conn.authorizers[-1] - assert final_authorizer is None or final_authorizer() == sqlite3.SQLITE_OK + assert conn.authorizers[-1] is None diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 45eea483..88ce5520 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -3,16 +3,14 @@ Tests for the write_wrapper plugin hook. """ import asyncio -import sqlite3 -import time from dataclasses import dataclass - -import pytest - from datasette.app import Datasette from datasette.events import Event from datasette.hookspecs import hookimpl from datasette.plugins import pm +import pytest +import sqlite3 +import time @dataclass @@ -115,8 +113,7 @@ async def test_write_wrapper_exception_thrown_into_generator(datasette): def wrapper(conn): try: yield - except Exception as e: # noqa: BLE001 - # Test helper deliberately captures whatever the wrapped write raised + except Exception as e: caught["error"] = e return wrapper @@ -235,6 +232,7 @@ async def test_write_wrapper_return_none_skips(datasette): @hookimpl def write_wrapper(datasette, database, request, transaction): log.append("hook-called") + return None pm.register(Plugin(), name="test_skip") try: @@ -341,7 +339,7 @@ async def test_write_wrapper_via_api(tmp_path): "/test/api_test/-/insert", json={"row": {"name": "test"}, "return": True}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", }, ) @@ -351,73 +349,6 @@ async def test_write_wrapper_via_api(tmp_path): pm.unregister(name="test_api") -@pytest.mark.asyncio -@pytest.mark.parametrize("num_sql_threads", (0, 1)) -@pytest.mark.parametrize("in_memory", (True, False)) -@pytest.mark.parametrize( - "operations", - ( - [{"op": "add_column", "args": {"name": "extra", "type": "text"}}], - [{"op": "rename_column", "args": {"name": "id", "to": "renamed_id"}}], - [{"op": "rename_table", "args": {"to": "renamed_t"}}], - [ - {"op": "add_column", "args": {"name": "extra", "type": "text"}}, - {"op": "rename_column", "args": {"name": "id", "to": "renamed_id"}}, - {"op": "rename_table", "args": {"to": "renamed_t"}}, - ], - ), - ids=["add-column", "transform", "rename-table", "combined"], -) -async def test_write_wrapper_can_reject_alter_table_after_write( - tmp_path, num_sql_threads, in_memory, operations -): - """Raising after yield should roll back the schema change.""" - db_path = str(tmp_path / "demo.db") - ds = Datasette( - [] if in_memory else [db_path], - config={"permissions": {"alter-table": True}}, - settings={"num_sql_threads": num_sql_threads}, - ) - db = ( - ds.add_memory_database(db_path, name="demo") - if in_memory - else ds.get_database("demo") - ) - await db.execute_write("CREATE TABLE t (id)") - await db.execute_write("INSERT INTO t (id) VALUES (1)") - before = await db.execute_fn(lambda conn: list(conn.iterdump())) - - class Plugin: - __name__ = "Plugin" - - @staticmethod - @hookimpl - def write_wrapper(database): - def wrapper(conn): - yield - raise ValueError("Rejected after write") - - return wrapper if database == "demo" else None - - pm.register(Plugin(), name="test_reject_alter_table") - try: - response = await ds.client.post( - "/demo/t/-/alter", - json={"operations": operations}, - ) - assert response.status_code == 400 - assert response.json()["errors"] == ["Rejected after write"] - assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before - assert not [ - event - for event in getattr(ds, "_tracked_events", []) - if event.name in ("alter-table", "rename-table") - ] - finally: - pm.unregister(name="test_reject_alter_table") - ds.close() - - @pytest.mark.asyncio async def test_write_wrapper_change_group_pattern(datasette): """Test the motivating use case: activating a change group around a write.""" @@ -535,7 +466,7 @@ async def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny try: request = FakeRequest(actor) if should_deny: - with pytest.raises(sqlite3.DatabaseError, match="not authorized"): + with pytest.raises(Exception): await db.execute_write_fn( lambda conn: conn.execute( f"insert into {table} (value) values ('test')"
      {i}{}{i}{i}a{i}