diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml deleted file mode 100644 index fdbc71c9..00000000 --- a/.github/actions/setup-sqlite-version/action.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Setup SQLite version" -description: "Build and activate a specific SQLite version from its amalgamation archive" -inputs: - version: - description: "The SQLite version to install" - required: true - cflags: - description: "CFLAGS to use when compiling SQLite" - required: false - default: "" - skip-activate: - description: "Set to true to skip modifying the library path" - required: false - default: "false" - fallback-urls: - description: "Whitespace-separated fallback download URLs to try after sqlite.org" - required: false - default: "" -outputs: - sqlite-location: - description: "Directory containing the compiled SQLite library" - value: ${{ steps.build.outputs.sqlite-location }} -runs: - using: "composite" - steps: - - shell: bash - run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" - - uses: actions/cache@v6 - with: - path: ${{ runner.temp }}/sqlite-versions/downloads - key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 - - id: build - shell: bash - run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" - env: - SQLITE_VERSION: ${{ inputs.version }} - SQLITE_CFLAGS: ${{ inputs.cflags }} - SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} - SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh deleted file mode 100644 index 03d6a68f..00000000 --- a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" -cflags="${SQLITE_CFLAGS:-}" -skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" -extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" - -case "$version_spec" in - 3.46 | 3.46.0) - sqlite_version="3.46.0" - sqlite_year="2024" - amalgamation_id="3460000" - builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" - ;; - 3.25 | 3.25.0) - sqlite_version="3.25.0" - sqlite_year="2018" - amalgamation_id="3250000" - builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1" - ;; - *) - echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." - exit 1 - ;; -esac - -case "$(uname -s)" in - Linux) - library_name="libsqlite3.so.0" - library_path_var="LD_LIBRARY_PATH" - ;; - Darwin) - library_name="libsqlite3.dylib" - library_path_var="DYLD_LIBRARY_PATH" - ;; - *) - echo "::error::Unsupported platform $(uname -s)" - exit 1 - ;; -esac - -runner_temp="${RUNNER_TEMP:-}" -if [ -z "$runner_temp" ]; then - runner_temp="$(mktemp -d)" -fi - -filename="sqlite-amalgamation-${amalgamation_id}" -official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" -download_dir="${runner_temp}/sqlite-versions/downloads" -source_root="${runner_temp}/sqlite-versions/source" -source_dir="${source_root}/${filename}" -build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" -archive_path="${download_dir}/${filename}.zip" - -mkdir -p "$download_dir" "$source_root" "$build_dir" - -download_archive() { - local url - local candidate_path="${archive_path}.tmp" - local urls=("$official_url") - - for url in $builtin_fallback_urls $extra_fallback_urls; do - urls+=("$url") - done - - rm -f "$candidate_path" - for url in "${urls[@]}"; do - echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" - if curl \ - --fail \ - --location \ - --show-error \ - --retry 5 \ - --retry-delay 2 \ - --retry-max-time 180 \ - --retry-all-errors \ - --connect-timeout 20 \ - --max-time 240 \ - --output "$candidate_path" \ - "$url"; then - mv "$candidate_path" "$archive_path" - return 0 - fi - - echo "::warning::Download failed from ${url}" - rm -f "$candidate_path" - done - - echo "::error::Could not download SQLite ${sqlite_version} amalgamation" - return 1 -} - -if [ ! -f "${source_dir}/sqlite3.c" ]; then - if [ ! -f "$archive_path" ]; then - download_archive - fi - - rm -rf "$source_dir" - unzip -q "$archive_path" -d "$source_root" -fi - -if [ ! -f "${source_dir}/sqlite3.c" ]; then - echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" - exit 1 -fi - -read -r -a cflag_args <<< "$cflags" - -echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" -gcc \ - -fPIC \ - -shared \ - "${cflag_args[@]}" \ - "${source_dir}/sqlite3.c" \ - "-I${source_dir}" \ - -o "${build_dir}/${library_name}" - -if [ "$library_name" = "libsqlite3.so.0" ]; then - ln -sf "$library_name" "${build_dir}/libsqlite3.so" -fi - -if [ -n "${GITHUB_OUTPUT:-}" ]; then - echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" -else - echo "sqlite-location=${build_dir}" -fi - -case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in - true | 1 | yes) - echo "Skipping ${library_path_var} activation" - ;; - *) - existing_value="${!library_path_var:-}" - if [ -n "${GITHUB_ENV:-}" ]; then - if [ -n "$existing_value" ]; then - echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" - else - echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" - fi - fi - echo "Added ${build_dir} to ${library_path_var}" - ;; -esac diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 3fc83438..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v7 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000..b8fb8aaa --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,16 @@ +name: Read the Docs Pull Request Preview +on: + pull_request: + types: + - opened + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "datasette" diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f5b8dbf6..5275ddef 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -16,7 +16,7 @@ jobs: matrix: browser: [chromium, firefox, webkit] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python 3.14 uses: actions/setup-python@v6 with: @@ -25,14 +25,14 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Cache uv - uses: actions/cache@v6 + uses: actions/cache@v5 with: path: ~/.cache/uv key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} restore-keys: | ${{ runner.os }}-py3.14-uv- - name: Cache Playwright browsers - uses: actions/cache@v6 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index d92ab82b..735e14e9 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v7 - - uses: actions/cache@v6 + uses: actions/checkout@v6 + - uses: actions/cache@v5 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 21ed4c12..87300593 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -56,7 +56,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index c5a4f0db..e622ef4c 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 58635025..9a808194 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index ecde5940..59b5fbc0 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@v6 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index e9bd4bab..c514048e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v7 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5e81ed82..5162c47a 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +20,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v6 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 2fdb3a40..23fce459 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,7 +25,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -34,7 +34,7 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: ./.github/actions/setup-sqlite-version + uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a8c0ae4..9e47db6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,17 +11,16 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index f2c074a6..a033cd92 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 5b8818c3..72af1eec 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: diff --git a/.gitignore b/.gitignore index 2a7f6620..8c058692 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,6 @@ datasets.json scratchpad -ignored/ - .vscode uv.lock diff --git a/Justfile b/Justfile index 6ffff870..5fcd9afd 100644 --- a/Justfile +++ b/Justfile @@ -33,11 +33,10 @@ export DATASETTE_SECRET := "not_a_secret" uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt uv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, ruff, prettier, cog +# Run linters: black, ruff, cog @lint: codespell uv run black datasette tests --check uv run ruff check datasette tests - npm run prettier -- --check uv run cog --check README.md docs/*.rst # Apply ruff fixes diff --git a/datasette/__init__.py b/datasette/__init__.py index e0022178..eb18e59e 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,14 +1,8 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa -from datasette.utils.asgi import ( # noqa - Forbidden, - NotFound, - PayloadTooLarge, - Request, - Response, -) +from datasette.tokens import TokenHandler, TokenRestrictions # noqa +from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 587380ed..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -89,8 +89,7 @@ def pytest_runtest_protocol(item, nextitem): continue try: ds.close() - except Exception as e: # noqa: BLE001 - # Surfaced as a pytest warning; teardown must not fail the run + except Exception as e: item.warn( pytest.PytestUnraisableExceptionWarning( f"Error closing Datasette instance: {e!r}" diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 7503f1d5..368213af 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,9 +1,7 @@ -import time - -from itsdangerous import BadSignature - from datasette import hookimpl +from itsdangerous import BadSignature from datasette.utils import baseconv +import time @hookimpl diff --git a/datasette/app.py b/datasette/app.py index 42be7425..9c9b7de4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,8 +2,7 @@ from __future__ import annotations import asyncio import contextvars -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence if TYPE_CHECKING: from datasette.permissions import Resource @@ -13,10 +12,11 @@ import dataclasses import datetime import functools import glob +import httpx import importlib.metadata import inspect +from itsdangerous import BadSignature import json -import logging import os import re import secrets @@ -28,36 +28,84 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx -from itsdangerous import BadSignature, URLSafeSerializer +from markupsafe import Markup, escape +from itsdangerous import URLSafeSerializer from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, - PrefixLoader, pass_context, + PrefixLoader, ) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -from markupsafe import Markup, escape -from . import stored_queries, write_sql -from .column_types import SQLiteType -from .csrf import CrossOriginProtectionMiddleware -from .database import Database, QueryInterrupted from .events import Event -from .plugins import DEFAULT_PLUGINS, get_plugins, pm +from .column_types import SQLiteType +from . import stored_queries, write_sql +from .views import Context +from .views.database import ( + database_download, + DatabaseView, + QueryView, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + QueryEditView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) +from .views.index import IndexView +from .views.special import ( + JsonDataView, + PatternPortfolioView, + AutocompleteDebugView, + AuthTokenView, + ApiExplorerView, + CreateTokenView, + LogoutView, + AllowDebugView, + PermissionsDebugView, + MessagesDebugView, + AllowedResourcesView, + PermissionRulesView, + PermissionCheckView, + JumpView, + InstanceSchemaView, + DatabaseSchemaView, + TableSchemaView, +) +from .views.table import ( + TableAutocompleteView, + TableInsertView, + TableUpsertView, + TableSetColumnTypeView, + TableDropView, + TableFragmentView, + table_view, +) +from .views.row import RowView, RowDeleteView, RowUpdateView from .renderer import json_renderer -from .resources import DatabaseResource, TableResource -from .tokens import TokenInvalid -from .tracer import AsgiTracer from .url_builder import Urls +from .database import Database, QueryInterrupted + from .utils import ( - SPATIALITE_FUNCTIONS, PaginatedResources, PrefixedUrlString, + SPATIALITE_FUNCTIONS, StartupError, - add_cors_headers, async_call_with_supported_arguments, await_me_maybe, baseconv, @@ -72,97 +120,45 @@ from .utils import ( move_plugins_and_allow, move_table_config, parse_metadata, - redact_keys, resolve_env_secrets, resolve_routes, - row_sql_params_pks, sha256_file, tilde_decode, tilde_encode, to_css_class, urlsafe_components, + redact_keys, + row_sql_params_pks, ) from .utils.asgi import ( AsgiLifespan, - AsgiRunOnFirstRequest, - BadRequest, - DatabaseNotFound, Forbidden, NotFound, + DatabaseNotFound, + TableNotFound, + RowNotFound, Request, Response, - RowNotFound, - TableNotFound, + AsgiRunOnFirstRequest, + asgi_static, asgi_send, asgi_send_file, asgi_send_redirect, - asgi_static, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, ) +from .tracer import AsgiTracer +from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .views import Context -from .views.database import ( - DatabaseView, - QueryView, - database_download, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.index import IndexView -from .views.row import RowDeleteView, RowUpdateView, RowView -from .views.special import ( - AllowDebugView, - AllowedResourcesView, - ApiExplorerView, - AuthTokenView, - AutocompleteDebugView, - CreateTokenView, - DatabaseSchemaView, - InstanceSchemaView, - JsonDataView, - JumpView, - LogoutView, - MessagesDebugView, - PatternPortfolioView, - PermissionCheckView, - PermissionRulesView, - PermissionsDebugView, - TableSchemaView, -) -from .views.stored_queries import ( - GlobalQueryListView, - QueryCreateAnalyzeView, - QueryDefinitionView, - QueryDeleteView, - QueryEditView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) -from .views.table import ( - TableAutocompleteView, - TableDropView, - TableFragmentView, - TableInsertView, - TableSetColumnTypeView, - TableUpsertView, - table_view, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) + +from .resources import DatabaseResource, TableResource app_root = Path(__file__).parent.parent -logger = logging.getLogger(__name__) - # Context variable to track when code is executing within a datasette.client request _in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) @@ -185,7 +181,7 @@ class PermissionCheck: """Represents a logged permission check for debugging purposes.""" when: str - actor: dict[str, Any] | None + actor: Dict[str, Any] | None action: str parent: str | None child: str | None @@ -210,11 +206,6 @@ SETTINGS = ( 100, "Maximum rows that can be inserted at a time using the bulk insert API", ), - Setting( - "max_post_body_bytes", - 2 * 1024 * 1024, - "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit", - ), Setting( "num_sql_threads", 3, @@ -435,7 +426,7 @@ class Datasette: if config_dir: db_files = [] for ext in ("db", "sqlite", "sqlite3"): - db_files.extend(config_dir.glob(f"*.{ext}")) + db_files.extend(config_dir.glob("*.{}".format(ext))) self.files += tuple(str(f) for f in db_files) if ( config_dir @@ -453,10 +444,8 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this - self._setup_db_done = False try: self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -464,7 +453,6 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() else: raise self.crossdb = crossdb @@ -679,10 +667,10 @@ class Datasette: def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: - for hook_environment in pm.hook.jinja2_environment_from_request( + for environment in pm.hook.jinja2_environment_from_request( datasette=self, request=request, env=environment ): - environment = hook_environment + pass return environment def get_action(self, name_or_abbr: str): @@ -736,7 +724,7 @@ class Datasette: catalog_database_names.update( row["database_name"] for row in await internal_db.execute( - f"select distinct database_name from {table}" + "select distinct database_name from {}".format(table) ) if row["database_name"] is not None ) @@ -747,7 +735,7 @@ class Datasette: for stale_db_name in stale_databases: for table in catalog_table_names: conn.execute( - f"DELETE FROM {table} WHERE database_name = ?", + "DELETE FROM {} WHERE database_name = ?".format(table), [stale_db_name], ) @@ -757,7 +745,19 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - await populate_schema_tables(internal_db, db, schema_version) + placeholders = "(?, ?, ?, ?)" + values = [database_name, str(db.path), db.is_memory, schema_version] + if db.path is None: + placeholders = "(?, null, ?, ?)" + values = [database_name, db.is_memory, schema_version] + await internal_db.execute_write( + """ + INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) + VALUES {} + """.format(placeholders), + values, + ) + await populate_schema_tables(internal_db, db) @property def urls(self): @@ -796,13 +796,17 @@ class Datasette: action.name in action_names and action != action_names[action.name] ): - raise StartupError(f"Duplicate action name: {action.name}") + raise StartupError( + "Duplicate action name: {}".format(action.name) + ) if ( action.abbr and action.abbr in action_abbrs and action != action_abbrs[action.abbr] ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") + raise StartupError( + "Duplicate action abbr: {}".format(action.abbr) + ) action_names[action.name] = action if action.abbr: action_abbrs[action.abbr] = action @@ -861,7 +865,7 @@ class Datasette: actor_id: str, *, expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + restrictions: "TokenRestrictions | None" = None, handler: str | None = None, ) -> str: """ @@ -901,9 +905,7 @@ class Datasette: Verify an API token by trying all registered token handlers. Returns an actor dict from the first handler that recognizes the - token, or None if no handler accepts it. A handler may raise - TokenInvalid for a token it recognizes but rejects (bad signature, - expired) - Datasette turns that into a 401 response. + token, or None if no handler accepts it. """ for token_handler in self._token_handlers(): result = await token_handler.verify_token(self, token) @@ -918,7 +920,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - name = next(iter(self.databases.keys())) + name = [key for key in self.databases.keys()][0] return self.databases[name] def add_database(self, db, name=None, route=None): @@ -931,7 +933,7 @@ class Datasette: suggestion = name i = 2 while name in self.databases: - name = f"{suggestion}_{i}" + name = "{}_{}".format(suggestion, i) i += 1 db.name = name db.route = route or name @@ -966,14 +968,13 @@ class Datasette: for db in dbs: try: db.close() - except Exception as e: # noqa: BLE001 - # Collect the first failure and re-raise after every close() has run + except Exception as e: if first_exception is None: first_exception = e if self.executor is not None: try: self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: # noqa: BLE001 + except Exception as e: if first_exception is None: first_exception = e if first_exception is not None: @@ -1322,15 +1323,24 @@ class Datasette: actual = ( actual_sqlite_type.value if actual_sqlite_type is not None - else f"unrecognized {column_detail.type!r}" + else "unrecognized {!r}".format(column_detail.type) ) raise ValueError( - f"Column type {ct_cls.name!r} is only applicable to SQLite types {allowed} but {database}.{resource}.{column} " - f"has SQLite type {actual}" + "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " + "has SQLite type {}".format( + ct_cls.name, + allowed, + database, + resource, + column, + actual, + ) ) async def _apply_column_types_config(self): """Load column_types from datasette.json config into the internal DB.""" + import logging + for db_name, db_conf in (self.config or {}).get("databases", {}).items(): for table_name, table_conf in db_conf.get("tables", {}).items(): for col_name, ct in table_conf.get("column_types", {}).items(): @@ -1340,7 +1350,7 @@ class Datasette: col_type = ct["type"] config = ct.get("config") if col_type not in self._column_types: - logger.warning( + logging.warning( "column_types config references unknown type %r " "for %s.%s.%s", col_type, @@ -1353,7 +1363,7 @@ class Datasette: db_name, table_name, col_name, col_type, config ) except ValueError as ex: - logger.warning(str(ex)) + logging.warning(str(ex)) async def get_column_type(self, database: str, resource: str, column: str): """ @@ -1406,7 +1416,7 @@ class Datasette: resource: str, column: str, column_type: str, - config: dict | None = None, + config: dict = None, ) -> None: """Assign a column type. Overwrites any existing assignment.""" ct_cls = self._column_types.get(column_type) @@ -1490,7 +1500,9 @@ class Datasette: possible_names = {plugin["name"], plugin["name"].replace("-", "_")} if plugin_name in possible_names: return _resolve_static_asset_path(plugin["static_path"], path) - raise FileNotFoundError(f"No static assets found for plugin {plugin_name}") + raise FileNotFoundError( + "No static assets found for plugin {}".format(plugin_name) + ) def _static_mounted_asset(self, mount_name, path): mount_name = mount_name.strip("/") @@ -1500,7 +1512,7 @@ class Datasette: _resolve_static_asset_path(dirname, path), self.urls.path("/{}/{}".format(mount_name, path.lstrip("/"))), ) - raise FileNotFoundError(f"No static mount found for {mount_name}") + raise FileNotFoundError("No static mount found for {}".format(mount_name)) def _static_asset_hash(self, filepath): filepath = Path(filepath) @@ -1591,17 +1603,18 @@ class Datasette: if await self.allowed(action="view-instance", actor=actor): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link - if database and await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), - actor=actor, - ): - crumbs.append( - { - "href": self.urls.database(database), - "label": database, - } - ) + if database: + if await self.allowed( + action="view-database", + resource=DatabaseResource(database=database), + actor=actor, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) # Table link if table: assert database, "table= requires database=" @@ -1620,7 +1633,7 @@ class Datasette: async def actors_from_ids( self, actor_ids: Iterable[str | int] - ) -> dict[int | str, dict]: + ) -> Dict[int | str, Dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1629,9 +1642,9 @@ class Datasette: return result async def track_event(self, event: Event): - assert isinstance( - event, self.event_classes - ), f"Invalid event type: {type(event)}" + assert isinstance(event, self.event_classes), "Invalid event type: {}".format( + type(event) + ) for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) @@ -1668,7 +1681,7 @@ class Datasette: self, actor: dict, action: str, - resource: Resource | None = None, + resource: "Resource" | None = None, ): """ Check if actor can see a resource and if it's private. @@ -1867,7 +1880,10 @@ class Datasette: if truncated and resources: last_resource = resources[-1] # Use tilde-encoding like table pagination - next_token = f"{tilde_encode(str(last_resource.parent))},{tilde_encode(str(last_resource.child))}" + next_token = "{},{}".format( + tilde_encode(str(last_resource.parent)), + tilde_encode(str(last_resource.child)), + ) return PaginatedResources( resources=resources, @@ -1885,7 +1901,7 @@ class Datasette: self, *, action: str, - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ) -> bool: """ @@ -1916,7 +1932,7 @@ class Datasette: self, *, actions: Sequence[str], - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ) -> dict[str, bool]: """ @@ -1937,11 +1953,11 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ + from datasette.utils.actions_sql import check_permissions_for_actions from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, ) - from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None @@ -2030,7 +2046,7 @@ class Datasette: self, *, action: str, - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ): """ @@ -2084,15 +2100,13 @@ class Datasette: db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) # Find the foreign_key for this column - fk = next( - ( + try: + fk = [ foreign_key for foreign_key in foreign_keys if foreign_key["column"] == column - ), - None, - ) - if fk is None: + ][0] + except IndexError: return {} # Ensure user has permission to view the referenced table from datasette.resources import TableResource @@ -2154,18 +2168,6 @@ class Datasette: for name, d in self.databases.items() ] - async def _connected_databases_for_actor(self, actor): - page = await self.allowed_resources("view-database", actor) - allowed_names = {resource.parent async for resource in page.all()} - return [ - database - for database in self._connected_databases() - if database["name"] in allowed_names - ] - - async def _databases_data(self, request): - return {"databases": await self._connected_databases_for_actor(request.actor)} - def _versions(self): conn = sqlite3.connect(":memory:") self._prepare_connection(conn, "_memory") @@ -2180,17 +2182,16 @@ class Datasette: sqlite_extensions[extension] = result.fetchone()[0] else: sqlite_extensions[extension] = None - except Exception: # noqa: BLE001, S110 - # Probing for optional SQLite extensions - absence is the normal case + except Exception: pass # More details on SpatiaLite if "spatialite" in sqlite_extensions: spatialite_details = {} for fn in SPATIALITE_FUNCTIONS: try: - result = conn.execute(f"select {fn}()") + result = conn.execute("select {}()".format(fn)) spatialite_details[fn] = result.fetchone()[0] - except sqlite3.Error as e: + except Exception as e: spatialite_details[fn] = {"error": str(e)} sqlite_extensions["spatialite"] = spatialite_details @@ -2198,7 +2199,9 @@ class Datasette: fts_versions = [] for fts in ("FTS5", "FTS4", "FTS3"): try: - conn.execute(f"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)") + conn.execute( + "CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(fts=fts) + ) fts_versions.append(fts) except sqlite3.OperationalError: continue @@ -2257,7 +2260,7 @@ class Datasette: "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": sorted(set(p["hooks"])), + "hooks": list(sorted(set(p["hooks"]))), } for p in ps ] @@ -2333,15 +2336,13 @@ class Datasette: async def render_template( self, - templates: list[str] | str | Template, - context: dict[str, Any] | Context | None = None, + templates: List[str] | str | Template, + context: Dict[str, Any] | Context | None = None, request: Request | None = None, view_name: str | None = None, ): if not self._startup_invoked: - raise RuntimeError( - "render_template() called before await ds.invoke_startup()" - ) + raise Exception("render_template() called before await ds.invoke_startup()") context = context or {} if isinstance(templates, Template): template = templates @@ -2387,9 +2388,9 @@ class Datasette: datasette=self, ): extra_vars = await await_me_maybe(extra_vars) - assert isinstance( - extra_vars, dict - ), f"extra_vars is of type {type(extra_vars)}" + assert isinstance(extra_vars, dict), "extra_vars is of type {}".format( + type(extra_vars) + ) extra_template_vars.update(extra_vars) async def menu_links(): @@ -2408,27 +2409,29 @@ class Datasette: # the contract tests fail otherwise template_context = { **context, - "request": request, - "crumb_items": self._crumb_items, - "urls": self.urls, - "actor": request.actor if request else None, - "menu_links": menu_links, - "display_actor": display_actor, - "show_logout": request is not None - and "ds_actor" in request.cookies - and request.actor, - "zip": zip, - "body_scripts": body_scripts, - "format_bytes": format_bytes, - "show_messages": lambda: self._show_messages(request), - "extra_css_urls": await self._asset_urls( - "extra_css_urls", template, context, request, view_name - ), - "extra_js_urls": await self._asset_urls( - "extra_js_urls", template, context, request, view_name - ), - "base_url": self.setting("base_url"), - "datasette_version": __version__, + **{ + "request": request, + "crumb_items": self._crumb_items, + "urls": self.urls, + "actor": request.actor if request else None, + "menu_links": menu_links, + "display_actor": display_actor, + "show_logout": request is not None + and "ds_actor" in request.cookies + and request.actor, + "zip": zip, + "body_scripts": body_scripts, + "format_bytes": format_bytes, + "show_messages": lambda: self._show_messages(request), + "extra_css_urls": await self._asset_urls( + "extra_css_urls", template, context, request, view_name + ), + "extra_js_urls": await self._asset_urls( + "extra_js_urls", template, context, request, view_name + ), + "base_url": self.setting("base_url"), + "datasette_version": __version__, + }, **extra_template_vars, } if request and request.args.get("_context") and self.setting("template_debug"): @@ -2511,8 +2514,8 @@ class Datasette: def add_route(view, regex): routes.append((regex, view)) - add_route(IndexView.as_view(self), r"/(\.(?Pjson))?$") - add_route(IndexView.as_view(self), r"/-/(\.(?Pjson))?$") + add_route(IndexView.as_view(self), r"/(\.(?Pjsono?))?$") + add_route(IndexView.as_view(self), r"/-/(\.(?Pjsono?))?$") add_route(permanent_redirect("/-/"), r"/-$") add_route(favicon, "/favicon.ico") @@ -2548,10 +2551,7 @@ class Datasette: ) add_route( JsonDataView.as_view( - self, - "plugins.json", - self._plugins, - needs_request=True, + self, "plugins.json", self._plugins, needs_request=True ), r"/-/plugins(\.(?Pjson))?$", ) @@ -2564,18 +2564,11 @@ class Datasette: r"/-/config(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view( - self, "threads.json", self._threads, permission="permissions-debug" - ), + JsonDataView.as_view(self, "threads.json", self._threads), r"/-/threads(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view( - self, - "databases.json", - self._databases_data, - needs_request=True, - ), + JsonDataView.as_view(self, "databases.json", self._connected_databases), r"/-/databases(\.(?Pjson))?$", ) add_route( @@ -2588,7 +2581,7 @@ class Datasette: JsonDataView.as_view( self, "actions.json", - lambda: {"actions": self._actions()}, + self._actions, template="debug_actions.html", permission="permissions-debug", ), @@ -2796,62 +2789,30 @@ class Datasette: db, table_name, _ = await self.resolve_table(request) pk_values = urlsafe_components(request.url_vars["pks"]) sql, params, pks = await row_sql_params_pks(db, table_name, pk_values) - if len(pk_values) != len(pks): - raise BadRequest( - "URL row identifier does not match the primary key for this table" - ) results = await db.execute(sql, params, truncate=True) row = results.first() if row is None: raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) - async def _startup_sequence(self): - """Idempotently run the full startup sequence: table counts for - immutable databases, then invoke_startup(). Safe to call more than - once and safe to call concurrently - callers block until whichever - call got there first has finished. - - This is the single entry point used by both AsgiLifespan (so - real deployments finish startup before accepting requests) and - AsgiRunOnFirstRequest (the fallback for hosts that never send - lifespan events, e.g. DatasetteClient's httpx.ASGITransport), and - `datasette serve` (cli.py) calls it too. The fast path below checks - both `_startup_invoked` and `_setup_db_done` - not just the former - - so that a bare `await ds.invoke_startup()` made by a caller ahead of - `_startup_sequence()` (which only sets `_startup_invoked`) can't - make this method skip the immutable-database table-count precompute. - """ - if self._startup_invoked and self._setup_db_done: - return - async with self._startup_lock: - if self._startup_invoked and self._setup_db_done: - return - if not self._setup_db_done: - # First time server starts up, calculate table counts for - # immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - self._setup_db_done = True - await self.invoke_startup() - def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() + async def setup_db(): + # First time server starts up, calculate table counts for immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + async def _close_on_shutdown(): self.close() asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan( - asgi, - on_startup=[self._startup_sequence], - on_shutdown=[_close_on_shutdown], - ) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[self._startup_sequence]) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) return asgi @@ -2886,11 +2847,7 @@ class DatasetteRouter: if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] scope = dict(scope, route_path=path) - request = Request( - scope, - receive, - max_post_body_bytes=self.ds.setting("max_post_body_bytes"), - ) + request = Request(scope, receive) # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -2910,24 +2867,13 @@ class DatasetteRouter: # Handle authentication default_actor = scope.get("actor") or None actor = None - token_error = None results = pm.hook.actor_from_request(datasette=self.ds, request=request) for result in results: - try: - result = await await_me_maybe(result) - except TokenInvalid as ex: - # A presented token was recognized but rejected - fail the - # request with a 401 even if another credential is valid, - # but keep awaiting the remaining coroutines first - if token_error is None: - token_error = ex - continue + result = await await_me_maybe(result) if result and actor is None: actor = result # Don't break — we must await all coroutines to avoid # "coroutine was never awaited" warnings - if token_error is not None: - return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) @@ -2956,19 +2902,9 @@ class DatasetteRouter: custom_response ), "Default forbidden() hook should have been called" return await custom_response.asgi_send(send) - except Exception as exception: # noqa: BLE001 - # This IS the top-level error handler - it must catch everything + except Exception as exception: return await self.handle_exception(request, send, exception) - async def handle_401(self, request, send, exception): - # A presented bearer token was recognized by a handler but rejected. - # Bearer tokens are API credentials, so this is always JSON. - headers = {"www-authenticate": 'Bearer error="invalid_token"'} - if self.ds.cors: - add_cors_headers(headers) - response = Response.error([str(exception)], 401, headers=headers) - await response.asgi_send(send) - async def handle_404(self, request, send, exception=None): # If path contains % encoding, redirect to tilde encoding if "%" in request.path: @@ -2979,7 +2915,7 @@ class DatasetteRouter: request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E") ) if request.query_string: - new_path += f"?{request.query_string}" + new_path += "?{}".format(request.query_string) await asgi_send_redirect(send, new_path) return # If URL has a trailing slash, redirect to URL without it @@ -3189,7 +3125,8 @@ _curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): # Drop the ".html" suffix - filepath = filepath.removesuffix(".html") + if filepath.endswith(".html"): + filepath = filepath[: -len(".html")] re_bits = ["/"] for bit in _curly_re.split(filepath): if _curly_re.match(bit): diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index b6c8b77f..4d8c6bea 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -1,8 +1,7 @@ -import hashlib - from datasette import hookimpl +from datasette.utils.asgi import Response, BadRequest from datasette.utils import to_css_class -from datasette.utils.asgi import BadRequest, Response +import hashlib _BLOB_COLUMN = "_blob_column" _BLOB_HASH = "_blob_hash" diff --git a/datasette/cli.py b/datasette/cli.py index 2694c1f6..90a33e80 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -1,45 +1,43 @@ import asyncio +import uvicorn +import click +from click import formatting +from click.types import CompositeParamType +from click_default_group import DefaultGroup import functools import json import os import pathlib +from runpy import run_module import shutil +from subprocess import call import sys import textwrap import webbrowser -from runpy import run_module -from subprocess import call - -import click -import uvicorn -from click import formatting -from click.types import CompositeParamType -from click_default_group import DefaultGroup - from .app import ( + Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, - Datasette, pm, ) from .inspect import inspect_tables from .utils import ( - ConnectionProblem, LoadExtension, - SpatialiteConnectionProblem, - SpatialiteNotFound, StartupError, - StaticMount, - ValueAsBooleanError, check_connection, deep_dict_update, find_spatialite, + parse_metadata, + ConnectionProblem, + SpatialiteConnectionProblem, initial_path_for_datasette, pairs_to_nested_config, - parse_metadata, temporary_docker_directory, value_as_boolean, + SpatialiteNotFound, + StaticMount, + ValueAsBooleanError, ) from .utils.sqlite import sqlite3 from .utils.testing import TestClient @@ -77,7 +75,7 @@ class Setting(CompositeParamType): # Datasette 1.0, we turn bare setting names into setting.name # Type checking for those older settings default = DEFAULT_SETTINGS[name] - name = f"settings.{name}" + name = "settings.{}".format(name) if isinstance(default, bool): try: return name, "true" if value_as_boolean(value) else "false" @@ -173,6 +171,7 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" + pass # Register publish plugins @@ -579,27 +578,27 @@ def serve( # https://github.com/simonw/datasette/issues/2389 deep_dict_update(config_data, settings_updates) - kwargs = { - "immutables": immutable, - "cache_headers": not reload, - "cors": cors, - "inspect_data": inspect_data, - "config": config_data, - "metadata": metadata_data, - "sqlite_extensions": sqlite_extensions, - "template_dir": template_dir, - "plugins_dir": plugins_dir, - "static_mounts": static, - "settings": None, # These are passed in config= now - "memory": memory, - "secret": secret, - "version_note": version_note, - "pdb": pdb, - "crossdb": crossdb, - "nolock": nolock, - "internal": internal, - "default_deny": default_deny, - } + kwargs = dict( + immutables=immutable, + cache_headers=not reload, + cors=cors, + inspect_data=inspect_data, + config=config_data, + metadata=metadata_data, + sqlite_extensions=sqlite_extensions, + template_dir=template_dir, + plugins_dir=plugins_dir, + static_mounts=static, + settings=None, # These are passed in config= now + memory=memory, + secret=secret, + version_note=version_note, + pdb=pdb, + crossdb=crossdb, + nolock=nolock, + internal=internal, + default_deny=default_deny, + ) # Separate directories from files directories = [f for f in files if os.path.isdir(f)] @@ -622,7 +621,9 @@ def serve( conn.close() else: raise click.ClickException( - f"Invalid value for '[FILES]...': Path '{file}' does not exist." + "Invalid value for '[FILES]...': Path '{}' does not exist.".format( + file + ) ) # Check for duplicate files by resolving all paths to their absolute forms @@ -663,6 +664,16 @@ def serve( # Private utility mechanism for writing unit tests return ds + # Run async soundness checks before startup hooks, since invoke_startup + # now populates internal tables which requires querying each database + run_sync(lambda: check_databases(ds)) + + # Run the "startup" plugin hooks + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -670,18 +681,10 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: - # --get means we don't run Uvicorn at all - run_sync(lambda: check_databases(ds)) - - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - client = TestClient(ds) request_headers = {} if token: - request_headers["Authorization"] = f"Bearer {token}" + request_headers["Authorization"] = "Bearer {}".format(token) cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) @@ -702,54 +705,30 @@ def serve( sys.exit(exit_code) return - # check_databases, invoke_startup() and the uvicorn server all run on a - # single event loop, so that anything a plugin's "startup" hook schedules - # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is - # still alive when the server starts handling requests. - async def _serve_async(): - # Populate internal catalog tables before invoke_startup - await check_databases(ds) - - # Run the full startup sequence (immutable-database table-count - # precompute + the "startup" plugin hooks) via the same entry point - # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when - # uvicorn's lifespan.startup fires moments later. - try: - await ds._startup_sequence() - except StartupError as e: - raise click.ClickException(e.args[0]) - - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = await initial_path_for_datasette(ds) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) - await server.serve() - - asyncio.run(_serve_async()) + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = run_sync(lambda: initial_path_for_datasette(ds)) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = dict( + host=host, port=port, log_level="info", lifespan="on", workers=1 + ) + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + uvicorn.run(ds.app(), **uvicorn_kwargs) @cli.command() @@ -906,7 +885,7 @@ async def check_databases(ds): ) except ConnectionProblem as e: raise click.UsageError( - f"Connection to {database.path} failed check: {e.args[0]!s}" + f"Connection to {database.path} failed check: {str(e.args[0])}" ) # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning if ( @@ -914,5 +893,9 @@ async def check_databases(ds): and len([db for db in ds.databases.values() if not db.is_memory]) > SQLITE_LIMIT_ATTACHED ): - msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases" + msg = ( + "Warning: --crossdb only works with the first {} attached databases".format( + SQLITE_LIMIT_ATTACHED + ) + ) click.echo(click.style(msg, bold=True, fg="yellow"), err=True) diff --git a/datasette/column_types.py b/datasette/column_types.py index 92fdd969..11a14ec0 100644 --- a/datasette/column_types.py +++ b/datasette/column_types.py @@ -64,14 +64,14 @@ class ColumnType: Return an HTML string to render this cell value, or None to fall through to the default render_cell plugin hook chain. """ - return + return None async def validate(self, value, datasette): """ Validate a value before it is written. Return None if valid, or a string error message if invalid. """ - return + return None async def transform_value(self, value, datasette): """ diff --git a/datasette/csrf.py b/datasette/csrf.py index a62f9473..df239aee 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -40,12 +40,12 @@ def _origin_tuple(value): scheme = (parsed.scheme or "").lower() host = (parsed.hostname or "").lower() if not scheme or not host: - raise ValueError(f"missing scheme or host in {value!r}") + raise ValueError("missing scheme or host in {!r}".format(value)) port = parsed.port # may raise ValueError on bad ports if port is None: port = DEFAULT_PORTS.get(scheme) if port is None: - raise ValueError(f"unknown default port for scheme {scheme!r}") + raise ValueError("unknown default port for scheme {!r}".format(scheme)) return scheme, host, port @@ -125,7 +125,9 @@ class CrossOriginProtectionMiddleware: return await self._forbid( send, - f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'", + "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( + sec_fetch_site + ), ) return @@ -139,11 +141,11 @@ class CrossOriginProtectionMiddleware: request_scheme = self._request_scheme(scope) try: origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple(f"{request_scheme}://{host}") + expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) except ValueError: await self._forbid( send, - f"Malformed Origin {origin!r} or Host {host!r}", + "Malformed Origin {!r} or Host {!r}".format(origin, host), ) return @@ -153,7 +155,7 @@ class CrossOriginProtectionMiddleware: await self._forbid( send, - f"Origin {origin!r} does not match Host {host!r}", + "Origin {!r} does not match Host {!r}".format(origin, host), ) def _request_scheme(self, scope): @@ -161,8 +163,7 @@ class CrossOriginProtectionMiddleware: try: if self.datasette.setting("force_https_urls"): return "https" - except Exception: # noqa: BLE001, S110 - # Settings may not be readable this early; fall back to the ASGI scheme + except Exception: pass return scope.get("scheme") or "http" diff --git a/datasette/database.py b/datasette/database.py index e162d34e..e7fe1ed9 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,35 +1,33 @@ import asyncio import atexit +from collections import namedtuple import inspect import os +from pathlib import Path import queue +import sqlite_utils import sys import tempfile import threading import uuid -from collections import namedtuple -from pathlib import Path -import sqlite_utils - -from .inspect import inspect_hash from .tracer import trace from .utils import ( call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, - escape_sqlite, get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, - sqlite3, sqlite_timelimit, - table_column_details, + sqlite3, table_columns, + table_column_details, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables from .utils.sqlite import sqlite_hidden_table_names +from .inspect import inspect_hash connections = threading.local() @@ -100,7 +98,9 @@ class Database: def _check_not_closed(self): if self._closed: - raise DatasetteClosedError(f"Database {self.name!r} has been closed") + raise DatasetteClosedError( + "Database {!r} has been closed".format(self.name) + ) def _remove_pending_execute_future(self, future): with self._pending_execute_futures_lock: @@ -139,7 +139,7 @@ class Database: if write: extra_kwargs["isolation_level"] = "IMMEDIATE" if self.memory_name: - uri = f"file:{self.memory_name}?mode=memory&cache=shared" + uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) conn = sqlite3.connect( uri, uri=True, check_same_thread=False, **extra_kwargs ) @@ -192,20 +192,21 @@ class Database: write_thread.join(timeout=10) if write_thread.is_alive(): sys.stderr.write( - f"Datasette: write thread for {self.name!r} did not exit within 10s\n" + "Datasette: write thread for {!r} did not exit within 10s\n".format( + self.name + ) ) sys.stderr.flush() for future in pending_execute_futures: try: future.result() - except Exception: # noqa: BLE001, S110 - # Shutdown teardown - a failed pending write must not block close() + except Exception: pass # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: try: connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._all_file_connections = [] # Drop per-thread cached read connections we can reach @@ -217,13 +218,13 @@ class Database: if self._read_connection is not None: try: self._read_connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._read_connection = None if self._write_connection is not None: try: self._write_connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._write_connection = None if self.is_temp_disk: @@ -245,7 +246,6 @@ class Database: request=None, return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, - transaction=True, ): self._check_not_closed() if returning_limit < 0: @@ -258,9 +258,7 @@ class Database: ) with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn( - _inner, block=block, request=request, transaction=transaction - ) + results = await self.execute_write_fn(_inner, block=block, request=request) return results async def execute_write_script(self, sql, block=True, request=None): @@ -350,7 +348,6 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: - self._write_connection.execute("BEGIN IMMEDIATE") result = fn(self._write_connection) else: result = fn(self._write_connection) @@ -369,8 +366,7 @@ class Database: async def _dispatch_events_after_write(): try: await reply_future - except Exception: # noqa: BLE001 - # The write failed; skip success events regardless of why + except Exception: # if the write failed, don't emit success events return for event in pending_events: @@ -423,7 +419,9 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) - self._write_thread.name = f"_execute_writes for database {self.name}" + self._write_thread.name = "_execute_writes for database {}".format( + self.name + ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() @@ -444,8 +442,7 @@ class Database: try: conn = self.connect(write=True) self.ds._prepare_connection(conn, self.name) - except Exception as e: # noqa: BLE001 - # Stored and re-raised to whoever queues the next write + except Exception as e: conn_exception = e while True: task = self._write_queue.get() @@ -453,8 +450,7 @@ class Database: if conn is not None: try: conn.close() - except Exception: # noqa: BLE001, S110 - # Best-effort close as the write thread exits + except Exception: pass return exception = None @@ -473,21 +469,19 @@ class Database: except ValueError: # Was probably a memory connection pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e else: try: if task.transaction: with conn: - conn.execute("BEGIN IMMEDIATE") result = task.fn(conn) else: result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e _deliver_write_result(task, result, exception) @@ -554,7 +548,9 @@ class Database: raise QueryInterrupted(e, sql, params) if log_sql_errors: sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + "ERROR: conn={}, sql = {}, params = {}: {}\n".format( + conn, repr(sql), params, e + ) ) sys.stderr.flush() raise @@ -607,7 +603,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})", + f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", custom_time_limit=limit, ) ).rows[0][0] @@ -711,9 +707,9 @@ class Database: column_names and len(column_names) == 2 and ("id" in column_names or "pk" in column_names) - and set(column_names) != {"id", "pk"} + and not set(column_names) == {"id", "pk"} ): - return next(c for c in column_names if c not in ("id", "pk")) + return [c for c in column_names if c not in ("id", "pk")][0] # Couldn't find a label: return None @@ -855,10 +851,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( "fn", - "isolated_connection", + "task_id", "loop", "reply_future", - "task_id", + "isolated_connection", "transaction", ) @@ -899,7 +895,7 @@ class QueryInterrupted(Exception): self.params = params def __str__(self): - return f"QueryInterrupted: {self.e}" + return "QueryInterrupted: {}".format(self.e) class MultipleValues(Exception): diff --git a/datasette/default_actions.py b/datasette/default_actions.py index ee165ae5..2f78570b 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -2,8 +2,8 @@ from datasette import hookimpl from datasette.permissions import Action from datasette.resources import ( DatabaseResource, - QueryResource, TableResource, + QueryResource, ) @@ -61,12 +61,6 @@ def register_actions(): description="Create tables", resource_class=DatabaseResource, ), - Action( - name="create-view", - abbr="cv", - description="Create views", - resource_class=DatabaseResource, - ), Action( name="store-query", abbr="sq", @@ -117,12 +111,6 @@ def register_actions(): description="Drop tables", resource_class=TableResource, ), - Action( - name="drop-view", - abbr="dv", - description="Drop views", - resource_class=TableResource, - ), # Query-level actions (child-level) Action( name="view-query", diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index bff5f0a7..91c1c5aa 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -1,9 +1,8 @@ +from datasette import hookimpl import datetime import os import time -from datasette import hookimpl - def header(key, request): key = key.replace("_", "-").encode("utf-8") diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index dee5df42..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,29 +17,18 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -from .config import config_permissions_sql as config_permissions_sql -from .defaults import ( - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, -) -from .defaults import ( - default_action_permissions_sql as default_action_permissions_sql, -) -from .defaults import ( - # Avoid "datasette.default_permissions" does not explicitly export attribute - default_allow_sql_check as default_allow_sql_check, -) -from .defaults import ( - default_query_permissions_sql as default_query_permissions_sql, -) -from .restrictions import ( - ActorRestrictions as ActorRestrictions, -) - # Re-export all hooks and public utilities from .restrictions import ( actor_restrictions_sql as actor_restrictions_sql, -) -from .restrictions import ( restrictions_allow_action as restrictions_allow_action, + ActorRestrictions as ActorRestrictions, ) from .root import root_user_permissions_sql as root_user_permissions_sql +from .config import config_permissions_sql as config_permissions_sql +from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute + default_allow_sql_check as default_allow_sql_check, + default_action_permissions_sql as default_action_permissions_sql, + default_query_permissions_sql as default_query_permissions_sql, + DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, +) diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index 4494f07f..aab87c1c 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration. from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple if TYPE_CHECKING: from datasette.app import Datasette @@ -55,8 +55,8 @@ class ConfigPermissionProcessor: def __init__( self, - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, ): self.datasette = datasette @@ -74,8 +74,8 @@ class ConfigPermissionProcessor: self.restrictions = actor.get("_r", {}) if actor else {} # Pre-compute restriction info for efficiency - self.restricted_databases: set[str] = set() - self.restricted_tables: set[tuple[str, str]] = set() + self.restricted_databases: Set[str] = set() + self.restricted_tables: Set[Tuple[str, str]] = set() if self.has_restrictions: self.restricted_databases = { @@ -92,20 +92,16 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - def evaluate_allow_block(self, allow_block: Any) -> bool | None: + def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: """Evaluate an allow block against the current actor.""" if allow_block is None: return None - # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as - # integers, but should retain the CLI's boolean 1/0 behavior. - if isinstance(allow_block, int) and allow_block in (0, 1): - return bool(allow_block) return actor_matches_allow(self.actor, allow_block) def is_in_restriction_allowlist( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], ) -> bool: """Check if resource is allowed by actor restrictions.""" if not self.has_restrictions: @@ -147,9 +143,9 @@ class ConfigPermissionProcessor: def add_permissions_rule( self, - parent: str | None, - child: str | None, - permissions_block: dict | None, + parent: Optional[str], + child: Optional[str], + permissions_block: Optional[dict], scope_desc: str, ) -> None: """Add a rule from a permissions:{action} block.""" @@ -169,8 +165,8 @@ class ConfigPermissionProcessor: def add_allow_block_rule( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], allow_block: Any, scope_desc: str, ) -> None: @@ -202,8 +198,8 @@ class ConfigPermissionProcessor: def _add_restriction_gate_denies( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], is_allowed: bool, scope_desc: str, ) -> None: @@ -235,7 +231,7 @@ class ConfigPermissionProcessor: if db_name == parent: self.collector.add(db_name, table_name, False, reason) - def process(self) -> PermissionSQL | None: + def process(self) -> Optional[PermissionSQL]: """Process all config rules and return combined PermissionSQL.""" self._process_root_permissions() self._process_databases() @@ -425,10 +421,10 @@ class ConfigPermissionProcessor: @hookimpl(specname="permission_resources_sql") async def config_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> list[PermissionSQL] | None: +) -> Optional[List[PermissionSQL]]: """ Apply permission rules from datasette.yaml configuration. diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 6f97812b..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -29,28 +29,29 @@ DEFAULT_ALLOW_ACTIONS = frozenset( @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: """ Enforce the default_allow_sql setting. When default_allow_sql is false (the default), execute-sql is denied unless explicitly allowed by config or other rules. """ - if action == "execute-sql" and not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") + if action == "execute-sql": + if not datasette.setting("default_allow_sql"): + return PermissionSQL.deny(reason="default_allow_sql is false") return None @hookimpl(specname="permission_resources_sql") async def default_action_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: """ Provide default allow rules for standard view/execute actions. @@ -70,10 +71,10 @@ async def default_action_permissions_sql( @hookimpl(specname="permission_resources_sql") async def default_query_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: actor_id = actor.get("id") if isinstance(actor, dict) else None if action not in {"view-query", "update-query", "delete-query"}: diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py index 5e59b7b4..47e03569 100644 --- a/datasette/default_permissions/helpers.py +++ b/datasette/default_permissions/helpers.py @@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Set if TYPE_CHECKING: from datasette.app import Datasette @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datasette.permissions import PermissionSQL -def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: +def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: """ Get all name variants for an action (full name and abbreviation). @@ -27,7 +27,7 @@ def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: return variants -def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool: +def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: """Check if an action (or its abbreviation) is in a list.""" return bool(get_action_name_variants(datasette, action).intersection(action_list)) @@ -36,8 +36,8 @@ def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool class PermissionRow: """A single permission rule row.""" - parent: str | None - child: str | None + parent: Optional[str] + child: Optional[str] allow: bool reason: str @@ -46,14 +46,14 @@ class PermissionRowCollector: """Collects permission rows and converts them to PermissionSQL.""" def __init__(self, prefix: str = "row"): - self.rows: list[PermissionRow] = [] + self.rows: List[PermissionRow] = [] self.prefix = prefix def add( self, - parent: str | None, - child: str | None, - allow: bool | None, + parent: Optional[str], + child: Optional[str], + allow: Optional[bool], reason: str, if_not_none: bool = False, ) -> None: @@ -62,7 +62,7 @@ class PermissionRowCollector: return self.rows.append(PermissionRow(parent, child, allow, reason)) - def to_permission_sql(self) -> PermissionSQL | None: + def to_permission_sql(self) -> Optional[PermissionSQL]: """Convert collected rows to a PermissionSQL object.""" if not self.rows: return None diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index 88e1d274..a22cd7e5 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -8,7 +8,7 @@ contains allowlists of resources the actor can access. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Set, Tuple if TYPE_CHECKING: from datasette.app import Datasette @@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants class ActorRestrictions: """Parsed actor restrictions from the _r key.""" - global_actions: list[str] # _r.a - globally allowed actions + global_actions: List[str] # _r.a - globally allowed actions database_actions: dict # _r.d - {db_name: [actions]} table_actions: dict # _r.r - {db_name: {table: [actions]}} @classmethod - def from_actor(cls, actor: dict | None) -> ActorRestrictions | None: + def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: """Parse restrictions from actor dict. Returns None if no restrictions.""" if not actor: return None @@ -44,11 +44,11 @@ class ActorRestrictions: table_actions=restrictions.get("r", {}), ) - def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool: + def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: """Check if action is in the global allowlist.""" return action_in_list(datasette, action, self.global_actions) - def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]: + def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: """Get database names where this action is allowed.""" allowed = set() for db_name, db_actions in self.database_actions.items(): @@ -57,8 +57,8 @@ class ActorRestrictions: return allowed def get_allowed_tables( - self, datasette: Datasette, action: str - ) -> set[tuple[str, str]]: + self, datasette: "Datasette", action: str + ) -> Set[Tuple[str, str]]: """Get (database, table) pairs where this action is allowed.""" allowed = set() for db_name, tables in self.table_actions.items(): @@ -70,10 +70,10 @@ class ActorRestrictions: @hookimpl(specname="permission_resources_sql") async def actor_restrictions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> list[PermissionSQL] | None: +) -> Optional[List[PermissionSQL]]: """ Handle actor restriction-based permission rules. @@ -140,10 +140,10 @@ async def actor_restrictions_sql( def restrictions_allow_action( - datasette: Datasette, + datasette: "Datasette", restrictions: dict, action: str, - resource: str | tuple[str, str] | None, + resource: Optional[str | Tuple[str, str]], ) -> bool: """ Check if restrictions allow the requested action on the requested resource. diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py index 22d13f65..4931f7ff 100644 --- a/datasette/default_permissions/root.py +++ b/datasette/default_permissions/root.py @@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL @hookimpl(specname="permission_resources_sql") async def root_user_permissions_sql( - datasette: Datasette, - actor: dict | None, -) -> PermissionSQL | None: + datasette: "Datasette", + actor: Optional[dict], +) -> Optional[PermissionSQL]: """ Grant root user full permissions when --root flag is used. """ diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py index 52daf8a2..7a359dc6 100644 --- a/datasette/default_permissions/tokens.py +++ b/datasette/default_permissions/tokens.py @@ -7,7 +7,7 @@ to datasette.verify_token() so all registered handlers are tried. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -17,13 +17,15 @@ from datasette.tokens import SignedTokenHandler @hookimpl -def register_token_handler(datasette: Datasette): +def register_token_handler(datasette: "Datasette"): """Register the default signed token handler.""" return SignedTokenHandler() @hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None: +async def actor_from_signed_api_token( + datasette: "Datasette", request +) -> Optional[dict]: """ Authenticate requests using API tokens by delegating to all registered token handlers via datasette.verify_token(). diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py index 0f2f32ef..e41434ef 100644 --- a/datasette/default_table_actions.py +++ b/datasette/default_table_actions.py @@ -20,7 +20,7 @@ def table_actions(datasette, actor, database, table, request): "label": "Alter table", "description": "Change columns and primary key for this table.", "attrs": { - "aria-label": f"Alter table {table}", + "aria-label": "Alter table {}".format(table), "data-table-action": "alter-table", }, } diff --git a/datasette/events.py b/datasette/events.py index 5f3fd06e..e8786da9 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -1,8 +1,7 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone - from datasette.hookspecs import hookimpl +from datetime import datetime, timezone @dataclass diff --git a/datasette/extras.py b/datasette/extras.py index fb8c2e06..36014185 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -5,8 +5,6 @@ from typing import ClassVar from asyncinject import Registry -from datasette.utils.asgi import BadRequest - def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -115,17 +113,6 @@ class ExtraRegistry: self._allowed_names[key] = names return names - def validate_requested(self, requested, scope): - """ - Raise BadRequest if any requested extra name is not a public extra - for this scope. Used by data formats such as .json - HTML pages - silently ignore unknown names instead. - """ - allowed = self._allowed_names_for_scope(scope, include_internal=False) - unknown = sorted(name for name in requested if name not in allowed) - if unknown: - raise BadRequest("Unknown _extra: {}".format(", ".join(unknown))) - async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] diff --git a/datasette/facets.py b/datasette/facets.py index 8c09e1dc..abe0605e 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -1,13 +1,12 @@ import json import urllib - from datasette import hookimpl from datasette.database import QueryInterrupted from datasette.utils import ( - detect_json1, escape_sqlite, path_with_added_args, path_with_removed_args, + detect_json1, sqlite3, ) @@ -31,7 +30,7 @@ def load_facet_configs(request, table_config): assert ( len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, facet_config = next(iter(facet_config.items())) + type, facet_config = list(facet_config.items())[0] if isinstance(facet_config, str): facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( @@ -86,7 +85,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table - self.sql = sql or f"select * from {escape_sqlite(table)}" + self.sql = sql or f"select * from [{table}]" self.params = params or [] self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: @@ -161,13 +160,18 @@ class ColumnFacet(Facet): for column in columns: if column in already_enabled: continue - suggested_facet_sql = f""" - with limited as (select * from ({self.sql}) limit {self.suggest_consider}) - select {escape_sqlite(column)} as value, count(*) as n from limited + suggested_facet_sql = """ + with limited as (select * from ({sql}) limit {suggest_consider}) + select {column} as value, count(*) as n from limited where value is not null group by value - limit {facet_size + 1} - """ + limit {limit} + """.format( + column=escape_sqlite(column), + sql=self.sql, + limit=facet_size + 1, + suggest_consider=self.suggest_consider, + ) distinct_values = None try: distinct_values = await self.ds.execute( @@ -263,7 +267,7 @@ class ColumnFacet(Facet): for row in facet_rows: column_qs = column if column.startswith("_"): - column_qs = f"{column}__exact" + column_qs = "{}__exact".format(column) selected = (column_qs, str(row["value"])) in qs_pairs if selected: toggle_path = path_with_removed_args( @@ -338,12 +342,12 @@ class ArrayFacet(Facet): for v in await self.ds.execute( self.database, ( - f"select {escape_sqlite(column)} from ({self.sql}) " - f"where {escape_sqlite(column)} is not null " - f"and {escape_sqlite(column)} != '' " - f"and json_array_length({escape_sqlite(column)}) > 0 " + "select {column} from ({sql}) " + "where {column} is not null " + "and {column} != '' " + "and json_array_length({column}) > 0 " "limit 100" - ), + ).format(column=escape_sqlite(column), sql=self.sql), self.params, truncate=False, custom_time_limit=self.ds.setting( @@ -384,14 +388,14 @@ class ArrayFacet(Facet): source = source_and_config["source"] column = config.get("column") or config["simple"] # https://github.com/simonw/datasette/issues/448 - facet_sql = f""" - with inner as ({self.sql}), + facet_sql = """ + with inner as ({sql}), deduped_array_items as ( select distinct j.value, inner.* from - json_each([inner].{escape_sqlite(column)}) j + json_each([inner].{col}) j join inner ) select @@ -402,8 +406,12 @@ class ArrayFacet(Facet): group by value order by - count(*) desc, value limit {facet_size + 1} - """ + count(*) desc, value limit {limit} + """.format( + col=escape_sqlite(column), + sql=self.sql, + limit=facet_size + 1, + ) try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 3cfb36e5..95cc5f37 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,11 +1,8 @@ -import json -from typing import ClassVar - from datasette import hookimpl from datasette.resources import DatabaseResource -from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError - +from datasette.utils.asgi import BadRequest +import json from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -102,9 +99,9 @@ def search_filters(request, database, table, datasette): fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), match_clause=( - f":search_{i}" + ":search_{}".format(i) if search_mode_raw - else f"escape_fts(:search_{i})" + else "escape_fts(:search_{})".format(i) ), ) ) @@ -137,11 +134,11 @@ def through_filters(request, database, table, datasette): value = through_data["value"] db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - fk_to_us = next( - (fk for fk in outgoing_foreign_keys if fk["other_table"] == table), - None, - ) - if fk_to_us is None: + try: + fk_to_us = [ + fk for fk in outgoing_foreign_keys if fk["other_table"] == table + ][0] + except IndexError: raise DatasetteError( "Invalid _through - could not find corresponding foreign key" ) @@ -209,14 +206,10 @@ class TemplatedFilter(Filter): if self.numeric and converted.isdigit(): converted = int(converted) if self.no_argument: - kwargs = {"c": _quote_sqlite_identifier(column)} + kwargs = {"c": column} converted = None else: - kwargs = { - "c": _quote_sqlite_identifier(column), - "p": f"p{param_counter}", - "t": _quote_sqlite_identifier(table), - } + kwargs = {"c": column, "p": f"p{param_counter}", "t": table} return self.sql_template.format(**kwargs), converted def human_clause(self, column, value): @@ -230,14 +223,6 @@ class TemplatedFilter(Filter): return template.format(c=column, v=value) -def _quote_sqlite_identifier(identifier): - # Preserve the historic always-quoted SQL generated by TemplatedFilter. - escaped = escape_sqlite(identifier) - if escaped == identifier: - return f'"{identifier}"' - return escaped - - class InFilter(Filter): key = "in" display = "in" @@ -279,56 +264,56 @@ class Filters: TemplatedFilter( "exact", "=", - "{c} = :{p}", + '"{c}" = :{p}', lambda c, v: "{c} = {v}" if v.isdigit() else '{c} = "{v}"', ), TemplatedFilter( "not", "!=", - "{c} != :{p}", + '"{c}" != :{p}', lambda c, v: "{c} != {v}" if v.isdigit() else '{c} != "{v}"', ), TemplatedFilter( "contains", "contains", - "{c} like :{p}", + '"{c}" like :{p}', '{c} contains "{v}"', format="%{}%", ), TemplatedFilter( "notcontains", "does not contain", - "{c} not like :{p}", + '"{c}" not like :{p}', '{c} does not contain "{v}"', format="%{}%", ), TemplatedFilter( "endswith", "ends with", - "{c} like :{p}", + '"{c}" like :{p}', '{c} ends with "{v}"', format="%{}", ), TemplatedFilter( "startswith", "starts with", - "{c} like :{p}", + '"{c}" like :{p}', '{c} starts with "{v}"', format="{}%", ), - TemplatedFilter("gt", ">", "{c} > :{p}", "{c} > {v}", numeric=True), + TemplatedFilter("gt", ">", '"{c}" > :{p}', "{c} > {v}", numeric=True), TemplatedFilter( - "gte", "\u2265", "{c} >= :{p}", "{c} \u2265 {v}", numeric=True + "gte", "\u2265", '"{c}" >= :{p}', "{c} \u2265 {v}", numeric=True ), - TemplatedFilter("lt", "<", "{c} < :{p}", "{c} < {v}", numeric=True), + TemplatedFilter("lt", "<", '"{c}" < :{p}', "{c} < {v}", numeric=True), TemplatedFilter( - "lte", "\u2264", "{c} <= :{p}", "{c} \u2264 {v}", numeric=True + "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True ), - TemplatedFilter("like", "like", "{c} like :{p}", '{c} like "{v}"'), + TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'), TemplatedFilter( - "notlike", "not like", "{c} not like :{p}", '{c} not like "{v}"' + "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"' ), - TemplatedFilter("glob", "glob", "{c} glob :{p}", '{c} glob "{v}"'), + TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'), InFilter(), NotInFilter(), ] @@ -337,13 +322,13 @@ class Filters: TemplatedFilter( "arraycontains", "array contains", - """:{p} in (select value from json_each({t}.{c}))""", + """:{p} in (select value from json_each([{t}].[{c}]))""", '{c} contains "{v}"', ), TemplatedFilter( "arraynotcontains", "array does not contain", - """:{p} not in (select value from json_each({t}.{c}))""", + """:{p} not in (select value from json_each([{t}].[{c}]))""", '{c} does not contain "{v}"', ), ] @@ -351,34 +336,36 @@ class Filters: else [] ) + [ - TemplatedFilter("date", "date", "date({c}) = :{p}", '"{c}" is on date {v}'), TemplatedFilter( - "isnull", "is null", "{c} is null", "{c} is null", no_argument=True + "date", "date", 'date("{c}") = :{p}', '"{c}" is on date {v}' + ), + TemplatedFilter( + "isnull", "is null", '"{c}" is null', "{c} is null", no_argument=True ), TemplatedFilter( "notnull", "is not null", - "{c} is not null", + '"{c}" is not null', "{c} is not null", no_argument=True, ), TemplatedFilter( "isblank", "is blank", - "({c} is null or {c} = '')", + '("{c}" is null or "{c}" = "")', "{c} is blank", no_argument=True, ), TemplatedFilter( "notblank", "is not blank", - "({c} is not null and {c} != '')", + '("{c}" is not null and "{c}" != "")', "{c} is not blank", no_argument=True, ), ] ) - _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters} + _filters_by_key = {f.key: f for f in _filters} def __init__(self, pairs): self.pairs = pairs diff --git a/datasette/fixtures.py b/datasette/fixtures.py index 049e35ed..7c85e16a 100644 --- a/datasette/fixtures.py +++ b/datasette/fixtures.py @@ -1,10 +1,9 @@ +from datasette.utils.sqlite import sqlite3 +from datasette.utils import documented import itertools import random import string -from datasette.utils import documented -from datasette.utils.sqlite import sqlite3 - __all__ = [ "EXTRA_DATABASE_SQL", "TABLES", @@ -347,7 +346,9 @@ CREATE VIEW searchable_view_configured_by_metadata AS + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + "\n".join( [ - f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");' + 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + a=a, b=b, c=c, content=content + ) for a, b, c, content in generate_compound_rows(1001) ] ) diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 67bf0d8b..41c48396 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,20 +1,9 @@ -from datasette import Response, hookimpl - -from .utils import add_cors_headers +from datasette import hookimpl, Response @hookimpl(trylast=True) def forbidden(datasette, request, message): async def inner(): - if ( - request.path.split("?")[0].endswith(".json") - or "application/json" in (request.headers.get("accept") or "") - or request.headers.get("content-type") == "application/json" - ): - headers = {} - if datasette.cors: - add_cors_headers(headers) - return Response.error(message, 403, headers=headers) return Response.html( await datasette.render_template( "error.html", diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index c36d5dbe..2b311644 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,21 +1,16 @@ -import traceback - -from markupsafe import Markup - -from datasette import Response, hookimpl - -from .utils import add_cors_headers, error_body +from datasette import hookimpl, Response +from .utils import add_cors_headers from .utils.asgi import ( Base400, ) from .views.base import DatasetteError +from markupsafe import Markup +import traceback -# Debugger imports are deliberate - they back the "pdb" setting, which drops -# into a debugger on unhandled exceptions try: - import ipdb as pdb # noqa: T100 + import ipdb as pdb except ImportError: - import pdb # noqa: T100 + import pdb try: import rich @@ -33,7 +28,6 @@ def handle_exception(datasette, request, exception): rich.get_console().print_exception(show_locals=True) title = None - plain_message = None if isinstance(exception, Base400): status = exception.status info = {} @@ -42,7 +36,6 @@ def handle_exception(datasette, request, exception): status = exception.status info = exception.error_dict message = exception.message - plain_message = exception.plain_message if exception.message_is_html: message = Markup(message) title = exception.title @@ -52,13 +45,6 @@ def handle_exception(datasette, request, exception): message = str(exception) traceback.print_exc() templates = [f"{status}.html", "error.html"] - headers = {} - if datasette.cors: - add_cors_headers(headers) - if request.path.split("?")[0].endswith(".json"): - body = dict(info) - body.update(error_body(plain_message or message, status)) - return Response.json(body, status=status, headers=headers) info.update( { "ok": False, @@ -67,18 +53,24 @@ def handle_exception(datasette, request, exception): "title": title, } ) - environment = datasette.get_jinja_environment(request) - template = environment.select_template(templates) - return Response.html( - await template.render_async( - dict( - info, - urls=datasette.urls, - menu_links=list, - ) - ), - status=status, - headers=headers, - ) + headers = {} + if datasette.cors: + add_cors_headers(headers) + if request.path.split("?")[0].endswith(".json"): + return Response.json(info, status=status, headers=headers) + else: + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) + return Response.html( + await template.render_async( + dict( + info, + urls=datasette.urls, + menu_links=lambda: [], + ) + ), + status=status, + headers=headers, + ) return inner diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index f89f2f36..7c56f882 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -1,4 +1,5 @@ -from pluggy import HookimplMarker, HookspecMarker +from pluggy import HookimplMarker +from pluggy import HookspecMarker hookspec = HookspecMarker("datasette") hookimpl = HookimplMarker("datasette") diff --git a/datasette/inspect.py b/datasette/inspect.py index b126ce5c..5e681e03 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -1,13 +1,13 @@ import hashlib from .utils import ( + detect_spatialite, detect_fts, detect_primary_keys, - detect_spatialite, escape_sqlite, get_all_foreign_keys, - sqlite3, table_columns, + sqlite3, ) HASH_BLOCK_SIZE = 1024 * 1024 @@ -95,10 +95,10 @@ def inspect_tables(conn, database_metadata): """) ] - for t, table_info in tables.items(): + for t in tables.keys(): for hidden_table in hidden_tables: if t == hidden_table or t.startswith(hidden_table): - table_info["hidden"] = True + tables[t]["hidden"] = True continue return tables diff --git a/datasette/jump.py b/datasette/jump.py index d70d33df..d138e827 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -21,7 +21,7 @@ class JumpSQL: search_text: str | None = None, display_name: str | None = None, item_type: str = "menu", - ) -> JumpSQL: + ) -> "JumpSQL": if search_text is None: search_text = " ".join( text for text in (label, display_name, description) if text is not None diff --git a/datasette/permissions.py b/datasette/permissions.py index e03b065c..786dc026 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,7 +1,7 @@ -import contextvars from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple +import contextvars # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( @@ -72,8 +72,8 @@ class Resource(ABC): ) def __repr__(self) -> str: - return ( - f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})" + return "{}(parent={!r}, child={!r})".format( + self.__class__.__name__, self.parent, self.child ) @property @@ -129,6 +129,7 @@ class Resource(ABC): Must return two columns: parent, child """ + pass class AllowedResource(NamedTuple): diff --git a/datasette/plugins.py b/datasette/plugins.py index 9cf94079..ae2cb17d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,14 +1,20 @@ import importlib -import importlib.metadata as importlib_metadata -import importlib.resources as importlib_resources import os -import sys -from pprint import pprint - import pluggy - +from pprint import pprint +import sys from . import hookspecs +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -79,7 +85,7 @@ if DATASETTE_LOAD_PLUGINS is not None: # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) except importlib_metadata.PackageNotFoundError: - sys.stderr.write(f"Plugin {package_name} could not be found\n") + sys.stderr.write("Plugin {} could not be found\n".format(package_name)) # Load default plugins diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 9ace865b..63d22fe8 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,17 +1,15 @@ +from datasette import hookimpl +import click import json import os import re from subprocess import CalledProcessError, check_call, check_output -import click - -from datasette import hookimpl - -from ..utils import temporary_docker_directory from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) +from ..utils import temporary_docker_directory @hookimpl @@ -221,7 +219,7 @@ def publish_subcommand(publish): check_call( "gcloud builds submit --tag {}{}".format( - image_id, f" --timeout {timeout}" if timeout else "" + image_id, " --timeout {}".format(timeout) if timeout else "" ), shell=True, ) @@ -233,7 +231,7 @@ def publish_subcommand(publish): ("--min-instances", min_instances), ): if value is not None: - extra_deploy_options.append(f"{option} {value}") + extra_deploy_options.append("{} {}".format(option, value)) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, @@ -260,16 +258,24 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi ) from exc describe_cmd = ( - f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} " - f"--location {artifact_region} --quiet" + "gcloud artifacts repositories describe {repo} --project {project} " + "--location {location} --quiet" + ).format( + repo=artifact_repository, + project=artifact_project, + location=artifact_region, ) try: check_call(describe_cmd, shell=True) return except CalledProcessError: create_cmd = ( - f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker " - f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet' + "gcloud artifacts repositories create {repo} --repository-format=docker " + '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' + ).format( + repo=artifact_repository, + location=artifact_region, + project=artifact_project, ) try: check_call(create_cmd, shell=True) diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 27dfd4bf..29665eb3 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -1,11 +1,9 @@ +from ..utils import StaticMount +import click import os import shutil import sys -import click - -from ..utils import StaticMount - def add_common_publish_arguments_and_options(subcommand): for decorator in reversed( @@ -78,7 +76,9 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): """Exit (with error message) if ``binary` isn't installed""" if not shutil.which(binary): click.secho( - f"Publishing to {publish_target} requires {binary} to be installed and configured", + "Publishing to {publish_target} requires {binary} to be installed and configured".format( + publish_target=publish_target, binary=binary + ), bg="red", fg="white", bold=True, diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index b0290833..f576a346 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -1,21 +1,19 @@ +from contextlib import contextmanager +from datasette import hookimpl +import click import json import os import pathlib import shlex import shutil -import tempfile -from contextlib import contextmanager from subprocess import call, check_output - -import click - -from datasette import hookimpl -from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata +import tempfile from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) +from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata @hookimpl @@ -236,7 +234,7 @@ def temporary_heroku_directory( extras.extend(["--static", f"{mount_point}:{mount_point}"]) quoted_files = " ".join( - [f"-i {shlex.quote(file_name)}" for file_name in file_names] + ["-i {}".format(shlex.quote(file_name)) for file_name in file_names] ) procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) diff --git a/datasette/renderer.py b/datasette/renderer.py index 0e01f52f..f40e3dbb 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,13 +1,11 @@ import json - from datasette.extras import extra_names_from_request from datasette.utils import ( - CustomJSONEncoder, - error_body, - path_from_row_pks, - remove_infinites, - sqlite3, value_as_boolean, + remove_infinites, + CustomJSONEncoder, + path_from_row_pks, + sqlite3, ) from datasette.utils.asgi import Response @@ -54,7 +52,8 @@ def json_renderer(request, args, data, error, truncated=None): if error: shape = "objects" status_code = 400 - data.update(error_body(error, status_code)) + data["error"] = error + data["ok"] = False if truncated is not None: data["truncated"] = truncated @@ -88,8 +87,7 @@ def json_renderer(request, args, data, error, truncated=None): object_rows[pk_string] = row data = object_rows if shape_error: - status_code = 400 - data = error_body(shape_error, status_code) + data = {"ok": False, "error": shape_error} elif shape == "array": data = data["rows"] @@ -102,7 +100,12 @@ def json_renderer(request, args, data, error, truncated=None): data["rows"] = [list(row.values()) for row in data["rows"]] else: status_code = 400 - data = error_body(f"Invalid _shape: {shape}", status_code) + data = { + "ok": False, + "error": f"Invalid _shape: {shape}", + "status": 400, + "title": None, + } # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 diff --git a/datasette/static/app.css b/datasette/static/app.css index d101e4b7..ce800f61 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1641,11 +1641,6 @@ dialog.row-edit-dialog::backdrop { overflow-y: auto; } -.row-edit-fields[hidden], -.row-edit-bulk[hidden] { - display: none; -} - .row-edit-field { display: grid; grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); @@ -1705,118 +1700,6 @@ textarea.row-edit-input { background: var(--paper); } -.row-edit-binary-control { - display: grid; - gap: 8px; - box-sizing: border-box; - width: 100%; - min-width: 0; - border: 1px solid var(--rule); - border-radius: 5px; - padding: 10px; - background: #fff; -} - -.row-edit-binary-control:focus { - border-color: var(--accent); - outline: 3px solid rgba(26, 86, 219, 0.12); -} - -.row-edit-binary-preview[hidden] { - display: none; -} - -.row-edit-binary-preview img { - display: block; - max-width: min(240px, 100%); - max-height: 180px; - border: 1px solid var(--rule); - border-radius: 4px; - background: var(--paper); -} - -.row-edit-binary-status { - display: flex; - flex-wrap: wrap; - align-items: baseline; - gap: 8px; - min-width: 0; -} - -.row-edit-binary-size { - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.86rem; -} - -.row-edit-binary-name { - color: var(--muted); - font-size: 0.82rem; - overflow-wrap: anywhere; -} - -.row-edit-binary-name[hidden] { - display: none; -} - -.row-edit-binary-actions { - display: flex; - flex-wrap: wrap; - gap: 8px; -} - -.row-edit-binary-file-button, -.row-edit-binary-clear { - appearance: none; - border: 1px solid var(--rule); - border-radius: 4px; - background: #fff; - color: var(--accent); - cursor: pointer; - font: inherit; - font-size: 0.78rem; - line-height: 1.2; - padding: 6px 8px; -} - -.row-edit-binary-file-button:hover, -.row-edit-binary-file-button:focus-within, -.row-edit-binary-clear:hover, -.row-edit-binary-clear:focus { - background: #f8fafc; -} - -.row-edit-binary-file-button:focus-within, -.row-edit-binary-clear:focus { - outline: 3px solid rgba(26, 86, 219, 0.12); - outline-offset: 1px; -} - -.row-edit-binary-file-button input[type="file"] { - position: absolute; - width: 1px; - height: 1px; - opacity: 0; - overflow: hidden; -} - -.row-edit-binary-clear[hidden] { - display: none; -} - -.row-edit-binary-drop-target { - border: 1px dashed var(--rule); - border-radius: 4px; - padding: 7px 8px; - color: var(--muted); - font-size: 0.78rem; -} - -.row-edit-binary-dragover .row-edit-binary-drop-target { - border-color: var(--accent); - background: var(--paper); - color: var(--ink); -} - .row-edit-default { display: grid; grid-template-columns: minmax(0, 1fr) 7.25rem; @@ -1915,207 +1798,6 @@ textarea.row-edit-input { margin: 0; } -.row-edit-bulk { - display: grid; - gap: 8px; - padding: 16px 24px 24px; - overflow-y: auto; -} - -.row-edit-bulk-editor { - display: grid; - gap: 8px; -} - -.row-edit-bulk-editor[hidden] { - display: none; -} - -.row-edit-bulk-actions { - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 8px; - justify-content: flex-start; -} - -.row-edit-bulk-actions .btn { - padding-left: 12px; - padding-right: 12px; -} - -.row-edit-bulk-conflict { - display: grid; - grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); - gap: 8px 12px; - align-items: start; -} - -.row-edit-bulk-conflict[hidden] { - display: none; -} - -.row-edit-bulk-conflict-label { - color: var(--ink); - font-size: 0.82rem; - padding-top: 8px; -} - -.row-edit-bulk-conflict-control { - display: grid; - gap: 4px; -} - -.row-edit-bulk-conflict-help { - color: var(--muted); - font-size: 0.78rem; - margin: 0; -} - -.row-edit-copy-template-label-narrow { - display: none; -} - -.row-edit-bulk-template-note { - color: var(--muted); - font-size: 0.82rem; -} - -.row-edit-bulk-template-note-narrow { - display: none; -} - -.row-edit-bulk-textarea { - min-height: 16rem; - resize: vertical; - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.82rem; - line-height: 1.45; -} - -.row-edit-bulk-textarea.row-edit-bulk-drop-target { - border-color: var(--accent); - background: #f8fbff; - outline: 3px solid rgba(26, 86, 219, 0.12); -} - -.row-edit-bulk-note { - color: var(--muted); - font-size: 0.82rem; - margin: 0; -} - -.row-edit-bulk-note label, -.row-edit-bulk-note .button-as-link { - font: inherit; -} - -@media (max-width: 640px) { - .row-edit-copy-template-label-wide { - display: none; - } - - .row-edit-copy-template-label-narrow { - display: inline; - } - - .row-edit-bulk-template-note-wide { - display: none; - } - - .row-edit-bulk-template-note-narrow { - display: inline; - } -} - -.row-edit-bulk-preview { - display: grid; - gap: 8px; - margin-top: 8px; -} - -.row-edit-bulk-preview[hidden] { - display: none; -} - -.row-edit-bulk-preview-summary { - color: var(--ink); - font-size: 0.9rem; - font-weight: 600; - margin: 0; -} - -.row-edit-bulk-preview-table-wrap { - border: 1px solid var(--rule); - border-radius: 5px; - max-height: 18rem; - overflow: auto; - background: #fff; -} - -.row-edit-bulk-preview-table { - border-collapse: collapse; - font-size: 0.78rem; - min-width: 100%; - width: max-content; -} - -.row-edit-bulk-preview-table th, -.row-edit-bulk-preview-table td { - border-bottom: 1px solid var(--rule); - border-right: 1px solid var(--rule); - max-width: 18rem; - overflow-wrap: anywhere; - padding: 6px 8px; - text-align: left; - vertical-align: top; - white-space: normal; -} - -.row-edit-bulk-preview-table th { - background: var(--paper); - color: var(--ink); - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-weight: 600; - position: sticky; - top: 0; - z-index: 1; -} - -.row-edit-bulk-preview-table tr:last-child td { - border-bottom: none; -} - -.row-edit-bulk-preview-table th:last-child, -.row-edit-bulk-preview-table td:last-child { - border-right: none; -} - -.row-edit-bulk-preview-null, -.row-edit-bulk-preview-auto { - color: var(--muted); - font-style: italic; -} - -.row-edit-bulk-progress { - display: grid; - gap: 6px; -} - -.row-edit-bulk-progress[hidden] { - display: none; -} - -.row-edit-bulk-progress-bar { - width: 100%; -} - -.row-edit-bulk-progress-status { - color: var(--ink); - font-size: 0.9rem; - margin: 0; -} - datasette-autocomplete { display: block; position: relative; @@ -2194,16 +1876,6 @@ datasette-autocomplete input[type="text"], background: var(--paper); } -.row-edit-mode-link { - color: var(--accent); - font-size: 0.9rem; - margin-right: auto; -} - -.row-edit-mode-link[hidden] { - display: none; -} - .row-edit-dialog .btn { border: none; border-radius: 5px; @@ -2400,120 +2072,6 @@ select.table-create-input { gap: 10px; } -.table-create-columns[hidden], -.table-create-data[hidden], -.table-create-data-editor[hidden], -.table-create-data-preview[hidden] { - display: none; -} - -.table-create-data, -.table-create-data-editor { - display: grid; - gap: 8px; -} - -.table-create-data-label { - color: var(--ink); - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.82rem; -} - -.table-create-data-textarea { - min-height: 16rem; - resize: vertical; - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.82rem; - line-height: 1.45; -} - -.table-create-data-textarea.table-create-data-drop-target { - border-color: var(--accent); - background: #f8fbff; - outline: 3px solid rgba(26, 86, 219, 0.12); -} - -.table-create-data-note { - color: var(--muted); - font-size: 0.82rem; - margin: 0; -} - -.table-create-data-note label, -.table-create-data-note .button-as-link { - font: inherit; -} - -.table-create-data-preview { - display: grid; - gap: 10px; -} - -.table-create-data-preview-summary { - color: var(--ink); - font-size: 0.9rem; - font-weight: 600; - margin: 0; -} - -.table-create-data-pk-field { - display: grid; - grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); - gap: 12px; - align-items: center; -} - -.table-create-data-preview-table-wrap { - border: 1px solid var(--rule); - border-radius: 5px; - max-height: 18rem; - overflow: auto; - background: #fff; -} - -.table-create-data-preview-table { - border-collapse: collapse; - font-size: 0.78rem; - min-width: 100%; - width: max-content; -} - -.table-create-data-preview-table th, -.table-create-data-preview-table td { - border-bottom: 1px solid var(--rule); - border-right: 1px solid var(--rule); - max-width: 18rem; - overflow-wrap: anywhere; - padding: 6px 8px; - text-align: left; - vertical-align: top; - white-space: normal; -} - -.table-create-data-preview-table th { - background: var(--paper); - color: var(--ink); - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-weight: 600; - position: sticky; - top: 0; - z-index: 1; -} - -.table-create-data-preview-table tr:last-child td { - border-bottom: none; -} - -.table-create-data-preview-table th:last-child, -.table-create-data-preview-table td:last-child { - border-right: none; -} - -.table-create-data-preview-null { - color: var(--muted); - font-style: italic; -} - .table-create-column-list { display: grid; gap: 8px; @@ -2741,16 +2299,6 @@ select.table-create-input { background: var(--paper); } -.table-create-mode-link { - color: var(--accent); - font-size: 0.9rem; - margin-right: auto; -} - -.table-create-mode-link[hidden] { - display: none; -} - .table-create-dialog .btn { border: none; border-radius: 5px; @@ -3464,8 +3012,7 @@ select.table-alter-input { .row-edit-dialog .modal-header, .row-edit-summary, .row-edit-loading, - .row-edit-fields, - .row-edit-bulk { + .row-edit-fields { padding-left: 18px; padding-right: 18px; } @@ -3484,15 +3031,6 @@ select.table-alter-input { padding-top: 0; } - .row-edit-bulk-conflict { - grid-template-columns: 1fr; - gap: 5px; - } - - .row-edit-bulk-conflict-label { - padding-top: 0; - } - .row-edit-dialog .modal-footer { padding-left: 18px; padding-right: 18px; @@ -3524,11 +3062,6 @@ select.table-alter-input { padding-top: 0; } - .table-create-data-pk-field { - grid-template-columns: 1fr; - gap: 5px; - } - .table-create-column-headings { display: none; } diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 9e8b93f6..9f4f89b9 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -2,10 +2,8 @@ var ROW_DELETE_DIALOG_ID = "row-delete-dialog"; var rowDeleteDialogState = null; var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; -var ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES = 10 * 1024 * 1024; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; -var TABLE_CREATE_AUTOMATIC_PK = "__datasette_automatic_pk__"; var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; var tableAlterDialogState = null; @@ -812,60 +810,12 @@ async function loadTableCreateForeignKeyTargets(state) { ); } -function tableCreateIsDataMode(state) { - return state && state.mode === "data"; -} - -function tableCreateSaveButtonText(state) { - if (tableCreateIsDataMode(state)) { - return state.dataPreviewReady ? "Create table" : "Preview rows"; - } - return "Create table"; -} - -function tableCreateCanInsertRows() { - var data = databaseCreateTableData() || {}; - return !!data.canInsertRows; -} - -function syncTableCreateModeUi(state) { - if (!state) { - return; - } - var isDataMode = tableCreateIsDataMode(state); - state.columnsPanel.hidden = isDataMode; - state.dataPanel.hidden = !isDataMode; - state.dataEditor.hidden = !isDataMode || state.dataPreviewReady; - state.dataPreview.hidden = !isDataMode || !state.dataPreviewReady; - state.createFromDataLink.hidden = isDataMode || !tableCreateCanInsertRows(); - state.manualCreateLink.hidden = !isDataMode; -} - -function updateTableCreateDialogButtons(state) { - if (!state) { - return; - } - syncTableCreateModeUi(state); - state.cancelButton.disabled = state.isSaving; - state.saveButton.disabled = state.isSaving; - state.addColumnButton.disabled = state.isSaving; - state.cancelButton.textContent = - tableCreateIsDataMode(state) && state.dataPreviewReady ? "Back" : "Cancel"; - state.saveButton.textContent = state.isSaving - ? "Creating..." - : tableCreateSaveButtonText(state); -} - function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; } - var signature = { + return JSON.stringify({ table: state.tableName.value, - data: state.dataTextarea ? state.dataTextarea.value : "", - dataPrimaryKey: state.dataPkSelect - ? state.dataPkSelect.value - : TABLE_CREATE_AUTOMATIC_PK, columns: tableCreateDialogRows(state).map(function (row) { return { name: row.querySelector(".table-create-column-name").value, @@ -888,8 +838,7 @@ function tableCreateDialogSignature(state) { ).value || "", }; }), - }; - return JSON.stringify(signature); + }); } function tableCreateDialogHasChanges(state) { @@ -915,23 +864,18 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; + state.cancelButton.disabled = isSaving; + state.saveButton.disabled = isSaving; + state.addColumnButton.disabled = isSaving; + state.saveButton.textContent = isSaving ? "Creating..." : "Create table"; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { control.disabled = isSaving; }); - state.fields - .querySelectorAll( - ".table-create-data input, .table-create-data select, .table-create-data textarea, .table-create-data button", - ) - .forEach(function (control) { - control.disabled = isSaving; - }); - state.tableName.disabled = isSaving; if (!isSaving) { updateTableCreateColumnRules(state); } - updateTableCreateDialogButtons(state); updateTableCreateMoveButtons(state); } @@ -1287,11 +1231,8 @@ function addTableCreateColumn(state, column) { } function resetTableCreateDialog(state) { - state.mode = "manual"; state.nextColumnIndex = 0; state.tableName.value = ""; - state.dataTextarea.value = ""; - resetTableCreateDataPreview(state); state.columnList.textContent = ""; addTableCreateColumn(state, { name: "id", @@ -1304,39 +1245,9 @@ function resetTableCreateDialog(state) { primaryKey: false, }); updateTableCreateColumnRules(state); - updateTableCreateDialogButtons(state); state.initialSignature = tableCreateDialogSignature(state); } -function showTableCreateDataMode(state) { - if (!state || state.isSaving || !tableCreateCanInsertRows()) { - return; - } - state.mode = "data"; - clearTableCreateDialogError(state); - updateTableCreateDialogButtons(state); - if (state.dataPreviewReady && state.dataPkSelect) { - state.dataPkSelect.focus(); - } else { - state.dataTextarea.focus(); - } -} - -function showTableCreateManualMode(state) { - if (!state || state.isSaving) { - return; - } - state.mode = "manual"; - clearTableCreateDialogError(state); - updateTableCreateDialogButtons(state); - var firstInput = state.columnList.querySelector(".table-create-column-name"); - if (firstInput) { - firstInput.focus(); - } else { - state.tableName.focus(); - } -} - function collectTableCreatePayload(state) { var payload = { table: state.tableName.value.trim(), @@ -1404,9 +1315,14 @@ function collectTableCreateColumnTypeAssignments(state) { } function validateTableCreatePayload(payload) { - var tableNameError = validateTableCreateTableName(payload.table); - if (tableNameError) { - return tableNameError; + if (!payload.table) { + return "Table name is required."; + } + if (payload.table.indexOf("\n") !== -1) { + return "Table name cannot contain newlines."; + } + if (/^sqlite_/i.test(payload.table)) { + return "Table name cannot start with sqlite_."; } if (!payload.columns.length) { return "At least one column is required."; @@ -1436,19 +1352,6 @@ function validateTableCreatePayload(payload) { return null; } -function validateTableCreateTableName(tableName) { - if (!tableName) { - return "Table name is required."; - } - if (tableName.indexOf("\n") !== -1) { - return "Table name cannot contain newlines."; - } - if (/^sqlite_/i.test(tableName)) { - return "Table name cannot start with sqlite_."; - } - return null; -} - function validateTableCreateColumnTypeAssignments(assignments) { for (var i = 0; i < assignments.length; i += 1) { var assignment = assignments[i]; @@ -1471,476 +1374,6 @@ function validateTableCreateColumnTypeAssignments(assignments) { return null; } -function normalizeCreateTableDataJsonValue(value) { - if (typeof value === "undefined") { - return null; - } - if (Array.isArray(value) || (value && typeof value === "object")) { - return JSON.stringify(value); - } - return value; -} - -function createTableDataColumnObjects(names) { - return names.map(function (name) { - return { name: name }; - }); -} - -function validateCreateTableDataHeaders(headers) { - if (!headers.length) { - throw new Error("No columns found to preview."); - } - var seen = {}; - headers.forEach(function (name, index) { - if (!name) { - throw new Error("Column header " + (index + 1) + " is blank."); - } - if (name.indexOf("\n") !== -1) { - throw new Error("Column names cannot contain newlines."); - } - var key = name.toLowerCase(); - if (seen[key]) { - throw new Error("Duplicate column name: " + name); - } - seen[key] = true; - }); -} - -function jsonRowIsObject(item) { - return !!(item && typeof item === "object" && !Array.isArray(item)); -} - -function extractJsonObjectRows(parsed) { - if (Array.isArray(parsed)) { - return parsed; - } - if (!jsonRowIsObject(parsed)) { - throw new Error( - "JSON must be an array of objects, or an object containing an array of objects.", - ); - } - - var bestRows = null; - Object.keys(parsed).forEach(function (key) { - var value = parsed[key]; - if (!Array.isArray(value) || !value.every(jsonRowIsObject)) { - return; - } - if (!bestRows || value.length > bestRows.length) { - bestRows = value; - } - }); - if (!bestRows) { - throw new Error( - "JSON object must contain at least one root key with an array of objects.", - ); - } - return bestRows; -} - -function parseJsonObjectRows(text) { - var parsed; - try { - parsed = JSON.parse(text); - } catch (error) { - throw new Error("Invalid JSON: " + error.message); - } - var rows = extractJsonObjectRows(parsed); - if (!rows.length) { - throw new Error("No rows found to preview."); - } - return rows; -} - -function parseJsonCreateTableRows(text) { - var parsed = parseJsonObjectRows(text); - - var columnNames = []; - var columnMap = {}; - parsed.forEach(function (item, index) { - if (!item || typeof item !== "object" || Array.isArray(item)) { - throw new Error("JSON row " + (index + 1) + " must be an object."); - } - Object.keys(item).forEach(function (name) { - if (!columnMap[name]) { - columnMap[name] = true; - columnNames.push(name); - } - }); - }); - validateCreateTableDataHeaders(columnNames); - - var rows = parsed.map(function (item) { - var row = {}; - columnNames.forEach(function (name) { - row[name] = Object.prototype.hasOwnProperty.call(item, name) - ? normalizeCreateTableDataJsonValue(item[name]) - : null; - }); - return row; - }); - return { - columns: createTableDataColumnObjects(columnNames), - rows: rows, - }; -} - -function createTableDelimitedValueIsInteger(value) { - return /^[-+]?\d+$/.test(String(value).trim()); -} - -function createTableDelimitedValueIsFloat(value) { - var trimmed = String(value).trim(); - if (!trimmed) { - return false; - } - var numberValue = Number(trimmed); - return Number.isFinite(numberValue); -} - -function inferCreateTableDelimitedColumnType(values) { - var nonBlankValues = values.filter(function (value) { - return String(value).trim() !== ""; - }); - if (!nonBlankValues.length) { - return "text"; - } - if (nonBlankValues.every(createTableDelimitedValueIsInteger)) { - return "integer"; - } - if (nonBlankValues.every(createTableDelimitedValueIsFloat)) { - return "float"; - } - return "text"; -} - -function coerceCreateTableDelimitedValue(value, type) { - var trimmed = String(value).trim(); - if (trimmed === "") { - return type === "integer" || type === "float" ? null : ""; - } - if (type === "integer") { - return parseInt(trimmed, 10); - } - if (type === "float") { - return Number(trimmed); - } - return value; -} - -function detectCreateTableDataDelimiter(text) { - var firstLine = - text.split(/\r\n|\n|\r/).find(function (line) { - return line.trim() !== ""; - }) || ""; - var csvRows = delimiterPreviewRows(firstLine, ","); - var tsvRows = delimiterPreviewRows(firstLine, "\t"); - var csvColumns = csvRows.length ? csvRows[0].length : 0; - var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; - - if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { - return "\t"; - } - if (tsvColumns > csvColumns) { - return "\t"; - } - if (csvColumns > 1) { - return ","; - } - if (tsvColumns > 1) { - return "\t"; - } - return null; -} - -function parseDelimitedCreateTableRows(text) { - var delimiter = detectCreateTableDataDelimiter(text); - var rows = ( - delimiter === null - ? splitSingleColumnRows(text) - : splitDelimitedRows(text, delimiter) - ).filter(function (row) { - return !bulkInsertDelimitedRowIsBlank(row); - }); - if (!rows.length) { - throw new Error("No rows found to preview."); - } - - var headers = rows[0].map(function (value) { - return value.trim(); - }); - validateCreateTableDataHeaders(headers); - var dataRows = rows.slice(1); - if (!dataRows.length) { - throw new Error("No data rows found to preview."); - } - - dataRows.forEach(function (row, index) { - if (row.length > headers.length) { - throw new Error( - "Row " + - (index + 1) + - " has " + - row.length + - " values, but only " + - headers.length + - " columns were provided.", - ); - } - }); - - var columnTypes = headers.map(function (_name, columnIndex) { - return inferCreateTableDelimitedColumnType( - dataRows.map(function (row) { - return row[columnIndex] || ""; - }), - ); - }); - - return { - columns: createTableDataColumnObjects(headers), - rows: dataRows.map(function (row) { - var rowObject = {}; - headers.forEach(function (name, columnIndex) { - rowObject[name] = coerceCreateTableDelimitedValue( - row[columnIndex] || "", - columnTypes[columnIndex], - ); - }); - return rowObject; - }), - }; -} - -function parseCreateTableDataRows(text) { - var trimmed = text.trim(); - if (!trimmed) { - throw new Error("Paste rows before previewing."); - } - if (trimmed[0] === "[" || trimmed[0] === "{") { - return parseJsonCreateTableRows(trimmed); - } - return parseDelimitedCreateTableRows(trimmed); -} - -function tableCreateDataRecommendedPrimaryKey(columns, rows) { - var candidates = []; - columns.forEach(function (column, columnIndex) { - var distinctValues = {}; - var maxLength = 0; - var valid = rows.length > 0; - rows.forEach(function (row) { - if (!valid) { - return; - } - var value = row[column.name]; - if (value === null || typeof value === "undefined") { - valid = false; - return; - } - var text = String(value).trim(); - if (!text || text.length >= 20 || /\s/.test(text)) { - valid = false; - return; - } - if (distinctValues[text]) { - valid = false; - return; - } - distinctValues[text] = true; - maxLength = Math.max(maxLength, text.length); - }); - if (valid) { - candidates.push({ - name: column.name, - maxLength: maxLength, - columnIndex: columnIndex, - }); - } - }); - candidates.sort(function (left, right) { - if (left.maxLength !== right.maxLength) { - return left.maxLength - right.maxLength; - } - return left.columnIndex - right.columnIndex; - }); - return candidates.length ? candidates[0].name : TABLE_CREATE_AUTOMATIC_PK; -} - -function renderTableCreateDataPreview(state, preview) { - state.dataPreview.textContent = ""; - - var summary = document.createElement("p"); - summary.className = "table-create-data-preview-summary"; - summary.textContent = - "Previewing " + - preview.rows.length + - " row" + - (preview.rows.length === 1 ? "." : "s."); - state.dataPreview.appendChild(summary); - - var pkField = document.createElement("div"); - pkField.className = "table-create-data-pk-field"; - var pkLabel = document.createElement("label"); - pkLabel.className = "table-create-data-label"; - pkLabel.setAttribute("for", "table-create-data-primary-key"); - pkLabel.textContent = "Primary key"; - var pkSelect = document.createElement("select"); - pkSelect.id = "table-create-data-primary-key"; - pkSelect.className = "table-create-input table-create-data-primary-key"; - var automaticOption = document.createElement("option"); - automaticOption.value = TABLE_CREATE_AUTOMATIC_PK; - automaticOption.textContent = "Automatic ID column"; - pkSelect.appendChild(automaticOption); - preview.columns.forEach(function (column) { - var option = document.createElement("option"); - option.value = column.name; - option.textContent = column.name; - pkSelect.appendChild(option); - }); - pkSelect.value = tableCreateDataRecommendedPrimaryKey( - preview.columns, - preview.rows, - ); - pkSelect.addEventListener("change", function () { - clearTableCreateDialogError(state); - }); - state.dataPkSelect = pkSelect; - pkField.appendChild(pkLabel); - pkField.appendChild(pkSelect); - state.dataPreview.appendChild(pkField); - - var tableWrap = document.createElement("div"); - tableWrap.className = "table-create-data-preview-table-wrap"; - var table = document.createElement("table"); - table.className = "table-create-data-preview-table"; - var thead = document.createElement("thead"); - var headerRow = document.createElement("tr"); - preview.columns.forEach(function (column) { - var th = document.createElement("th"); - th.scope = "col"; - th.textContent = column.name; - headerRow.appendChild(th); - }); - thead.appendChild(headerRow); - table.appendChild(thead); - - var tbody = document.createElement("tbody"); - preview.rows.forEach(function (row) { - var tr = document.createElement("tr"); - preview.columns.forEach(function (column) { - var td = document.createElement("td"); - var value = row[column.name]; - td.textContent = bulkInsertPreviewValue(value); - if (value === null) { - td.className = "table-create-data-preview-null"; - } - tr.appendChild(td); - }); - tbody.appendChild(tr); - }); - table.appendChild(tbody); - tableWrap.appendChild(table); - state.dataPreview.appendChild(tableWrap); - state.dataPreview.hidden = false; -} - -function resetTableCreateDataPreview(state) { - state.dataPreviewRows = null; - state.dataPreviewColumns = []; - state.dataPreviewReady = false; - state.dataPkSelect = null; - state.dataPreview.hidden = true; - state.dataPreview.textContent = ""; - syncTableCreateModeUi(state); -} - -function tableCreateTableNameFromFileName(fileName) { - var baseName = (fileName || "").replace(/^.*[\\/]/, ""); - var nameWithoutExtension = baseName.replace(/\.[^.]*$/, ""); - return nameWithoutExtension - .trim() - .replace(/\s+/g, "_") - .toLowerCase() - .replace(/[^a-z0-9_]/g, ""); -} - -async function loadTableCreateDataTextFile(state, file) { - if (!file) { - return; - } - try { - var text = await readTextFile(file); - var tableName = tableCreateTableNameFromFileName(file.name); - if (tableName) { - state.tableName.value = tableName; - state.tableName.dispatchEvent(new Event("input", { bubbles: true })); - } - state.dataTextarea.value = text; - state.dataTextarea.dispatchEvent(new Event("input", { bubbles: true })); - clearTableCreateDialogError(state); - state.dataTextarea.focus(); - } catch (_error) { - showTableCreateDialogError(state, "Could not read that text file."); - } -} - -function previewTableCreateDataRows(state) { - clearTableCreateDialogError(state); - resetTableCreateDataPreview(state); - try { - var preview = parseCreateTableDataRows(state.dataTextarea.value); - state.dataPreviewRows = preview.rows; - state.dataPreviewColumns = preview.columns; - state.dataPreviewReady = true; - renderTableCreateDataPreview(state, preview); - updateTableCreateDialogButtons(state); - } catch (error) { - showTableCreateDialogError( - state, - error.message || "Could not preview rows.", - ); - updateTableCreateDialogButtons(state); - } -} - -function collectTableCreateDataPayload(state) { - var payload = { - table: state.tableName.value.trim(), - rows: state.dataPreviewRows || [], - }; - var primaryKey = state.dataPkSelect - ? state.dataPkSelect.value - : TABLE_CREATE_AUTOMATIC_PK; - payload.pk = - primaryKey === TABLE_CREATE_AUTOMATIC_PK ? "id" : primaryKey || undefined; - return payload; -} - -function validateTableCreateDataPayload(payload, state) { - var tableNameError = validateTableCreateTableName(payload.table); - if (tableNameError) { - return tableNameError; - } - if (!payload.rows.length) { - return "No rows found to create."; - } - if ( - state.dataPkSelect && - state.dataPkSelect.value === TABLE_CREATE_AUTOMATIC_PK && - payload.rows.some(function (row) { - return Object.prototype.hasOwnProperty.call(row, "id"); - }) - ) { - return ( - "Automatic ID column cannot be used because the pasted data " + - "already has an id column." - ); - } - return null; -} - function fallbackTableUrl(tableName) { var data = databaseCreateTableData() || {}; if (!data.path) { @@ -2008,57 +1441,6 @@ async function assignTableCreateColumnTypes( } } -async function createTableFromDataPreview(state) { - var data = databaseCreateTableData(); - if (!data || !data.path) { - showTableCreateDialogError(state, "Could not find the create table URL."); - return; - } - var payload = collectTableCreateDataPayload(state); - var validationError = validateTableCreateDataPayload(payload, state); - if (validationError) { - showTableCreateDialogError(state, validationError); - return; - } - clearTableCreateDialogError(state); - setTableCreateDialogSaving(state, true); - try { - var response = await fetch(data.path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var responseData = null; - try { - responseData = await response.json(); - } catch (_error) { - responseData = null; - } - if (!response.ok || (responseData && responseData.ok === false)) { - throw rowMutationRequestError(response, responseData); - } - var tableUrl = - responseData.table_url || - fallbackTableUrl(responseData.table || payload.table); - state.shouldRestoreFocus = false; - state.dialog.close(); - if (tableUrl) { - location.href = tableUrl; - } else { - location.reload(); - } - } catch (error) { - setTableCreateDialogSaving(state, false); - showTableCreateDialogError( - state, - error.message || "Could not create table", - ); - } -} - async function saveTableCreateDialog(state) { if (state.isSaving) { return; @@ -2068,14 +1450,6 @@ async function saveTableCreateDialog(state) { showTableCreateDialogError(state, "Could not find the create table URL."); return; } - if (tableCreateIsDataMode(state)) { - if (!state.dataPreviewReady) { - previewTableCreateDataRows(state); - } else { - await createTableFromDataPreview(state); - } - return; - } clearTableCreateDialogError(state); var payload = collectTableCreatePayload(state); var columnTypeAssignments = collectTableCreateColumnTypeAssignments(state); @@ -2186,18 +1560,8 @@ function ensureTableCreateDialog(manager) {
-
- Create table from data -
@@ -2212,27 +1576,13 @@ function ensureTableCreateDialog(manager) { error: dialog.querySelector(".table-create-error"), fields: dialog.querySelector(".table-create-fields"), tableName: dialog.querySelector(".table-create-table-name"), - columnsPanel: dialog.querySelector(".table-create-columns"), columnList: dialog.querySelector(".table-create-column-list"), addColumnButton: dialog.querySelector(".table-create-add-column"), - dataPanel: dialog.querySelector(".table-create-data"), - dataEditor: dialog.querySelector(".table-create-data-editor"), - dataTextarea: dialog.querySelector(".table-create-data-textarea"), - dataOpenFileButton: dialog.querySelector(".table-create-data-open-file"), - dataFileInput: dialog.querySelector(".table-create-data-file-input"), - dataPreview: dialog.querySelector(".table-create-data-preview"), - createFromDataLink: dialog.querySelector(".table-create-from-data"), - manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), currentButton: null, shouldRestoreFocus: true, isSaving: false, - mode: "manual", - dataPreviewRows: null, - dataPreviewColumns: [], - dataPreviewReady: false, - dataPkSelect: null, initialSignature: "", nextColumnIndex: 0, foreignKeyTargets: [], @@ -2256,114 +1606,13 @@ function ensureTableCreateDialog(manager) { }); tableCreateDialogState.cancelButton.addEventListener("click", function () { - if ( - tableCreateIsDataMode(tableCreateDialogState) && - tableCreateDialogState.dataPreviewReady && - !tableCreateDialogState.isSaving - ) { - resetTableCreateDataPreview(tableCreateDialogState); - updateTableCreateDialogButtons(tableCreateDialogState); - tableCreateDialogState.dataTextarea.focus(); - return; - } closeTableCreateDialogIfConfirmed(tableCreateDialogState); }); - tableCreateDialogState.createFromDataLink.addEventListener( - "click", - function (ev) { - ev.preventDefault(); - showTableCreateDataMode(tableCreateDialogState); - }, - ); - - tableCreateDialogState.manualCreateLink.addEventListener( - "click", - function (ev) { - ev.preventDefault(); - showTableCreateManualMode(tableCreateDialogState); - }, - ); - tableCreateDialogState.tableName.addEventListener("input", function () { clearTableCreateDialogError(tableCreateDialogState); }); - tableCreateDialogState.dataOpenFileButton.addEventListener( - "click", - function () { - tableCreateDialogState.dataFileInput.click(); - }, - ); - - tableCreateDialogState.dataFileInput.addEventListener( - "change", - async function (ev) { - var files = ev.target.files; - await loadTableCreateDataTextFile( - tableCreateDialogState, - files && files.length ? files[0] : null, - ); - ev.target.value = ""; - }, - ); - - tableCreateDialogState.dataTextarea.addEventListener( - "dragenter", - function (ev) { - ev.preventDefault(); - tableCreateDialogState.dataTextarea.classList.add( - "table-create-data-drop-target", - ); - }, - ); - - tableCreateDialogState.dataTextarea.addEventListener( - "dragover", - function (ev) { - ev.preventDefault(); - tableCreateDialogState.dataTextarea.classList.add( - "table-create-data-drop-target", - ); - }, - ); - - tableCreateDialogState.dataTextarea.addEventListener( - "dragleave", - function () { - tableCreateDialogState.dataTextarea.classList.remove( - "table-create-data-drop-target", - ); - }, - ); - - tableCreateDialogState.dataTextarea.addEventListener( - "drop", - async function (ev) { - ev.preventDefault(); - tableCreateDialogState.dataTextarea.classList.remove( - "table-create-data-drop-target", - ); - var files = ev.dataTransfer && ev.dataTransfer.files; - if (!files || !files.length) { - return; - } - await loadTableCreateDataTextFile(tableCreateDialogState, files[0]); - }, - ); - - tableCreateDialogState.dataTextarea.addEventListener("dragend", function () { - tableCreateDialogState.dataTextarea.classList.remove( - "table-create-data-drop-target", - ); - }); - - tableCreateDialogState.dataTextarea.addEventListener("input", function () { - resetTableCreateDataPreview(tableCreateDialogState); - clearTableCreateDialogError(tableCreateDialogState); - updateTableCreateDialogButtons(tableCreateDialogState); - }); - dialog.addEventListener("click", function (ev) { if (ev.target === dialog) { closeTableCreateDialogIfConfirmed(tableCreateDialogState); @@ -4333,14 +3582,6 @@ function tableInsertUrl() { return url.toString(); } -function tableUpsertUrl() { - var data = tableInsertData(); - if (data && data.upsertPath) { - return new URL(data.upsertPath, location.href).toString(); - } - return null; -} - function rowResourceUrl(row) { var rowId = row.getAttribute("data-row"); if (!rowId) { @@ -4357,7 +3598,7 @@ function rowJsonUrl(row) { return ""; } url.pathname = url.pathname + ".json"; - url.searchParams.set("_extra", "columns,column_types,column_details"); + url.searchParams.set("_extra", "columns,column_types"); return url.toString(); } @@ -4637,155 +3878,10 @@ function initRowDeleteActions(manager) { }); } -function isBase64JsonValue(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return false; - } - var keys = Object.keys(value); - return ( - keys.length === 2 && - Object.prototype.hasOwnProperty.call(value, "$base64") && - Object.prototype.hasOwnProperty.call(value, "encoded") && - value.$base64 === true && - typeof value.encoded === "string" - ); -} - -function shouldUseBinaryControl(value, options) { - options = options || {}; - var sqliteType = (options.sqliteType || "").toLowerCase(); - return ( - isBase64JsonValue(value) || - sqliteType === "blob" || - options.valueKind === "binary" - ); -} - -function binaryEncodedValue(value) { - return isBase64JsonValue(value) ? value.encoded : ""; -} - -function binaryByteLengthFromBase64(encoded) { - encoded = (encoded || "").replace(/\s/g, ""); - if (!encoded) { - return 0; - } - var padding = 0; - if (encoded.slice(-2) === "==") { - padding = 2; - } else if (encoded.slice(-1) === "=") { - padding = 1; - } - return Math.max(0, Math.floor((encoded.length * 3) / 4) - padding); -} - -function rowEditBinaryValue(encoded) { - return { - $base64: true, - encoded: encoded || "", - }; -} - -function formatRowEditBinarySize(byteLength) { - var number = byteLength.toLocaleString(); - return "Binary: " + number + " byte" + (byteLength === 1 ? "" : "s"); -} - -function base64ToUint8Array(encoded) { - var binary = window.atob(encoded || ""); - var bytes = new Uint8Array(binary.length); - for (var i = 0; i < binary.length; i += 1) { - bytes[i] = binary.charCodeAt(i); - } - return bytes; -} - -function uint8ArrayToBase64(bytes) { - var chunks = []; - var chunkSize = 0x8000; - for (var i = 0; i < bytes.length; i += chunkSize) { - chunks.push( - String.fromCharCode.apply(null, bytes.subarray(i, i + chunkSize)), - ); - } - return window.btoa(chunks.join("")); -} - -function rowEditBinaryImageMimeType(bytes) { - if (!bytes || !bytes.length) { - return null; - } - if ( - bytes.length >= 8 && - bytes[0] === 0x89 && - bytes[1] === 0x50 && - bytes[2] === 0x4e && - bytes[3] === 0x47 && - bytes[4] === 0x0d && - bytes[5] === 0x0a && - bytes[6] === 0x1a && - bytes[7] === 0x0a - ) { - return "image/png"; - } - if ( - bytes.length >= 3 && - bytes[0] === 0xff && - bytes[1] === 0xd8 && - bytes[2] === 0xff - ) { - return "image/jpeg"; - } - if ( - bytes.length >= 6 && - bytes[0] === 0x47 && - bytes[1] === 0x49 && - bytes[2] === 0x46 && - bytes[3] === 0x38 && - (bytes[4] === 0x37 || bytes[4] === 0x39) && - bytes[5] === 0x61 - ) { - return "image/gif"; - } - if ( - bytes.length >= 12 && - bytes[0] === 0x52 && - bytes[1] === 0x49 && - bytes[2] === 0x46 && - bytes[3] === 0x46 && - bytes[8] === 0x57 && - bytes[9] === 0x45 && - bytes[10] === 0x42 && - bytes[11] === 0x50 - ) { - return "image/webp"; - } - if ( - bytes.length >= 12 && - bytes[4] === 0x66 && - bytes[5] === 0x74 && - bytes[6] === 0x79 && - bytes[7] === 0x70 && - bytes[8] === 0x61 && - bytes[9] === 0x76 && - bytes[10] === 0x69 && - (bytes[11] === 0x66 || bytes[11] === 0x73) - ) { - return "image/avif"; - } - if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) { - return "image/bmp"; - } - return null; -} - function valueToEditText(value) { if (value === null || typeof value === "undefined") { return ""; } - if (isBase64JsonValue(value)) { - return value.encoded; - } if (typeof value === "object") { return JSON.stringify(value, null, 2); } @@ -4796,9 +3892,6 @@ function shouldUseTextarea(value, columnType) { if (columnType && columnType.type === "textarea") { return true; } - if (isBase64JsonValue(value)) { - return false; - } if (value && typeof value === "object") { return true; } @@ -4807,9 +3900,6 @@ function shouldUseTextarea(value, columnType) { } function rowEditValueKind(value) { - if (isBase64JsonValue(value)) { - return "binary"; - } if (value === null || typeof value === "undefined") { return "null"; } @@ -4833,261 +3923,6 @@ function rowEditControlElement(control, autocompleteUrl) { return autocomplete; } -function revokeRowEditBinaryPreview(wrapper) { - if (wrapper && wrapper._rowEditBinaryPreviewUrl) { - URL.revokeObjectURL(wrapper._rowEditBinaryPreviewUrl); - wrapper._rowEditBinaryPreviewUrl = null; - } -} - -function updateRowEditBinaryPreview(wrapper, encoded, byteLength) { - var preview = wrapper.querySelector(".row-edit-binary-preview"); - if (!preview) { - return; - } - revokeRowEditBinaryPreview(wrapper); - preview.hidden = true; - preview.textContent = ""; - if ( - !encoded || - byteLength >= ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES || - !window.atob || - !window.Blob || - !window.URL || - !URL.createObjectURL - ) { - return; - } - - var bytes; - try { - bytes = base64ToUint8Array(encoded); - } catch (_error) { - return; - } - var mimeType = rowEditBinaryImageMimeType(bytes); - if (!mimeType) { - return; - } - - var image = document.createElement("img"); - image.alt = ""; - var objectUrl = URL.createObjectURL(new Blob([bytes], { type: mimeType })); - wrapper._rowEditBinaryPreviewUrl = objectUrl; - - var showPreview = function () { - if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { - preview.hidden = false; - } - }; - var hidePreview = function () { - if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { - revokeRowEditBinaryPreview(wrapper); - preview.hidden = true; - preview.textContent = ""; - } - }; - image.onload = showPreview; - image.onerror = hidePreview; - image.src = objectUrl; - preview.appendChild(image); -} - -function updateRowEditBinaryDisplay(wrapper, control, fileName) { - var kind = rowEditControlValueKind(control); - var size = wrapper.querySelector(".row-edit-binary-size"); - var name = wrapper.querySelector(".row-edit-binary-name"); - var clear = wrapper.querySelector(".row-edit-binary-clear"); - var byteLength = - kind === "binary" ? binaryByteLengthFromBase64(control.value) : null; - - if (size) { - size.textContent = - kind === "binary" - ? formatRowEditBinarySize(byteLength) - : "No binary data"; - } - if (name) { - name.textContent = fileName || ""; - name.hidden = !fileName; - } - if (clear) { - clear.hidden = control.dataset.notNull === "1" || kind !== "binary"; - } - if (kind === "binary") { - updateRowEditBinaryPreview(wrapper, control.value, byteLength); - } else { - updateRowEditBinaryPreview(wrapper, "", 0); - } -} - -function setRowEditBinaryControlValue(control, encoded, fileName) { - control.value = encoded || ""; - control.dataset.currentValueKind = "binary"; - updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, fileName); - control.dispatchEvent(new Event("input", { bubbles: true })); -} - -function clearRowEditBinaryControlValue(control) { - control.value = ""; - control.dataset.currentValueKind = "null"; - updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, ""); - control.dispatchEvent(new Event("input", { bubbles: true })); -} - -function readRowEditBinaryFile(file) { - return new Promise(function (resolve, reject) { - var reader = new FileReader(); - reader.onload = function () { - try { - var bytes = new Uint8Array(reader.result || new ArrayBuffer(0)); - resolve({ - encoded: uint8ArrayToBase64(bytes), - name: file && file.name ? file.name : "", - }); - } catch (error) { - reject(error); - } - }; - reader.onerror = function () { - reject(reader.error || new Error("Could not read file")); - }; - reader.readAsArrayBuffer(file); - }); -} - -function rowEditBinaryValueFromText(text) { - var encoder = new TextEncoder(); - var bytes = encoder.encode(text || ""); - return { - encoded: uint8ArrayToBase64(bytes), - name: "Pasted text", - }; -} - -function rowEditBinaryFirstClipboardFile(clipboardData) { - if (!clipboardData) { - return null; - } - if (clipboardData.files && clipboardData.files.length) { - return clipboardData.files[0]; - } - var items = clipboardData.items || []; - for (var i = 0; i < items.length; i += 1) { - if (items[i].kind === "file") { - return items[i].getAsFile(); - } - } - return null; -} - -function handleRowEditBinaryFile(control, file) { - if (!file) { - return; - } - readRowEditBinaryFile(file) - .then(function (value) { - setRowEditBinaryControlValue(control, value.encoded, value.name); - }) - .catch(function (error) { - console.error("Could not read binary file", error); - }); -} - -function createRowEditBinaryControlElement(control, value, options, labelId) { - var wrapper = document.createElement("div"); - wrapper.className = "row-edit-binary-control"; - wrapper.dataset.column = control.name; - wrapper.setAttribute("role", "group"); - wrapper.setAttribute("tabindex", "0"); - wrapper.setAttribute("aria-labelledby", labelId); - - var preview = document.createElement("div"); - preview.className = "row-edit-binary-preview"; - preview.hidden = true; - - var status = document.createElement("div"); - status.className = "row-edit-binary-status"; - var size = document.createElement("span"); - size.className = "row-edit-binary-size"; - var name = document.createElement("span"); - name.className = "row-edit-binary-name"; - name.hidden = true; - status.appendChild(size); - status.appendChild(name); - - var actions = document.createElement("div"); - actions.className = "row-edit-binary-actions"; - var fileLabel = document.createElement("label"); - fileLabel.className = "row-edit-binary-file-button"; - fileLabel.textContent = "Attach file"; - var fileInput = document.createElement("input"); - fileInput.type = "file"; - fileInput.setAttribute("aria-label", "Attach file for " + control.name); - fileInput.addEventListener("change", function () { - if (fileInput.files && fileInput.files.length) { - handleRowEditBinaryFile(control, fileInput.files[0]); - } - }); - fileLabel.appendChild(fileInput); - actions.appendChild(fileLabel); - - var clearButton = document.createElement("button"); - clearButton.type = "button"; - clearButton.className = "row-edit-binary-clear"; - clearButton.textContent = "Set NULL"; - clearButton.addEventListener("click", function () { - clearRowEditBinaryControlValue(control); - wrapper.focus(); - }); - actions.appendChild(clearButton); - - var dropTarget = document.createElement("div"); - dropTarget.className = "row-edit-binary-drop-target"; - dropTarget.textContent = "Drop or paste file contents"; - ["dragenter", "dragover"].forEach(function (eventName) { - wrapper.addEventListener(eventName, function (ev) { - ev.preventDefault(); - wrapper.classList.add("row-edit-binary-dragover"); - }); - }); - ["dragleave", "drop"].forEach(function (eventName) { - wrapper.addEventListener(eventName, function () { - wrapper.classList.remove("row-edit-binary-dragover"); - }); - }); - wrapper.addEventListener("drop", function (ev) { - ev.preventDefault(); - var file = ev.dataTransfer && ev.dataTransfer.files[0]; - handleRowEditBinaryFile(control, file); - }); - wrapper.addEventListener("paste", function (ev) { - var file = rowEditBinaryFirstClipboardFile(ev.clipboardData); - if (file) { - ev.preventDefault(); - handleRowEditBinaryFile(control, file); - return; - } - var text = ev.clipboardData && ev.clipboardData.getData("text"); - if (text) { - ev.preventDefault(); - var pasted = rowEditBinaryValueFromText(text); - setRowEditBinaryControlValue(control, pasted.encoded, pasted.name); - } - }); - - control.type = "hidden"; - control._rowEditBinaryWrapper = wrapper; - wrapper.appendChild(control); - wrapper.appendChild(preview); - wrapper.appendChild(status); - wrapper.appendChild(actions); - wrapper.appendChild(dropTarget); - - updateRowEditBinaryDisplay(wrapper, control, ""); - return wrapper; -} - function columnTypeForContext(columnType) { if (!columnType) { return null; @@ -5351,11 +4186,6 @@ function focusFirstRowEditControl(state, options) { if (focusRowEditPluginControl(field)) { return true; } - var binaryControl = field.querySelector(".row-edit-binary-control"); - if (binaryControl) { - binaryControl.focus(); - return true; - } control.focus(); return true; } @@ -5379,11 +4209,6 @@ function destroyRowEditFields(state) { } } }); - state.fields - .querySelectorAll(".row-edit-binary-control") - .forEach(function (binaryControl) { - revokeRowEditBinaryPreview(binaryControl); - }); state.fields.innerHTML = ""; } @@ -5410,50 +4235,34 @@ function createRowEditField(column, value, isPk, columnType, index, options) { controlWrap.className = "row-edit-control-wrap"; var context = columnFormControlContext(column, isPk, columnType, options); - var isBinaryField = shouldUseBinaryControl(value, options); - var pluginControl = isBinaryField - ? null - : makeColumnField(options.manager, context); + var pluginControl = makeColumnField(options.manager, context); var useTextarea = - !isBinaryField && - ((pluginControl && pluginControl.useTextarea === true) || - shouldUseTextarea(value, columnType)); + (pluginControl && pluginControl.useTextarea === true) || + shouldUseTextarea(value, columnType); var control = useTextarea ? document.createElement("textarea") : document.createElement("input"); - var initialValue = isBinaryField - ? binaryEncodedValue(value) - : valueToEditText(value); - var initialValueKind = options.valueKind || rowEditValueKind(value); - if (isBinaryField) { - initialValueKind = isBase64JsonValue(value) ? "binary" : "null"; - } control.className = "row-edit-input"; control.id = fieldId; control.name = column; - control.value = initialValue; + control.value = valueToEditText(value); control.setAttribute("aria-describedby", metaId); - control.dataset.initialValue = initialValue; - control.dataset.initialValueKind = initialValueKind; + control.dataset.initialValue = valueToEditText(value); + control.dataset.initialValueKind = + options.valueKind || rowEditValueKind(value); control.dataset.primaryKey = isPk ? "1" : "0"; control.dataset.currentValueKind = control.dataset.initialValueKind; - if (isBinaryField) { - control.dataset.binaryField = "1"; - control.dataset.notNull = options.notnull ? "1" : "0"; - } if (hasDefaultExpression) { control.dataset.useSqliteDefault = useSqliteDefault ? "1" : "0"; } if (useSqliteDefault) { control.disabled = true; } - if (options.omitIfBlank || (isBinaryField && options.mode === "insert")) { + if (options.omitIfBlank) { control.dataset.omitIfBlank = "1"; } - if (isBinaryField) { - control.type = "hidden"; - } else if (control.nodeName === "TEXTAREA") { + if (control.nodeName === "TEXTAREA") { control.rows = Math.min(8, Math.max(3, control.value.split("\n").length)); } else { control.type = "text"; @@ -5527,11 +4336,9 @@ function createRowEditField(column, value, isPk, columnType, index, options) { field._datasetteColumnFormField = fieldApi; var pluginControlElement = renderColumnField(pluginControl, fieldApi); var controlElement = - (isBinaryField && - createRowEditBinaryControlElement(control, value, options, labelId)) || pluginControlElement || rowEditControlElement(control, options.autocompleteUrl); - if (options.autocompleteUrl && !pluginControlElement && !isBinaryField) { + if (options.autocompleteUrl && !pluginControlElement) { control.addEventListener("input", function () { setForeignKeyMetaLink(meta, options.autocompleteUrl, null); }); @@ -5622,64 +4429,18 @@ function clearRowEditDialogError(state) { state.error.textContent = ""; } -function showRowEditDialogError(state, message, options) { +function showRowEditDialogError(state, message) { state.error.hidden = false; state.error.textContent = message; - if (!options || options.focus !== false) { - state.error.focus(); - } -} - -function rowEditIsMultipleInsert(state) { - return state.mode === "insert" && state.insertMode === "multiple"; -} - -function syncRowEditInsertModeUi(state) { - var isInsert = state.mode === "insert"; - var isMultiple = rowEditIsMultipleInsert(state); - state.fields.hidden = isMultiple; - state.bulkInsertPanel.hidden = !isMultiple; - state.bulkInsertEditor.hidden = !isMultiple || state.bulkInsertPreviewReady; - state.bulkInsertPreview.hidden = !isMultiple || !state.bulkInsertPreviewReady; - state.bulkInsertLink.hidden = !isInsert || isMultiple; - state.singleInsertLink.hidden = !isInsert || !isMultiple; + state.error.focus(); } function updateRowEditDialogButtons(state) { state.saveButton.disabled = - state.isLoading || - state.isSaving || - !state.hasLoaded || - state.bulkInsertInserted || - (rowEditIsMultipleInsert(state) && - !state.bulkInsertPreviewReady && - !!state.bulkInsertLiveValidationError); + state.isLoading || state.isSaving || !state.hasLoaded; state.cancelButton.disabled = state.isSaving; - syncRowEditInsertModeUi(state); - state.cancelButton.textContent = - rowEditIsMultipleInsert(state) && - state.bulkInsertPreviewReady && - !state.bulkInsertInserted - ? "Back" - : state.bulkInsertInserted - ? "Close and view table" - : "Cancel"; - var saveLabel = rowEditIsMultipleInsert(state) - ? state.bulkInsertInserted - ? "Inserted" - : state.bulkInsertPreviewReady - ? bulkInsertSaveLabel(state) - : "Preview rows" - : state.mode === "insert" - ? "Insert row" - : "Save"; - state.saveButton.textContent = state.isSaving - ? rowEditIsMultipleInsert(state) - ? bulkInsertConflictMode(state) === "upsert" - ? "Updating..." - : "Inserting..." - : "Saving..." - : saveLabel; + var saveLabel = state.mode === "insert" ? "Insert row" : "Save"; + state.saveButton.textContent = state.isSaving ? "Saving..." : saveLabel; state.form.setAttribute( "aria-busy", state.isLoading || state.isSaving ? "true" : "false", @@ -5699,9 +4460,6 @@ function setRowEditDialogSaving(state, isSaving) { function valueFromRowEditControl(control) { var value = control.value; - if (rowEditControlValueKind(control) === "binary") { - return rowEditBinaryValue(value); - } return valueFromRowEditText( control.name, value, @@ -5712,9 +4470,6 @@ function valueFromRowEditControl(control) { function valueFromRowEditText(name, value, initialValueKind) { var trimmed = value.trim(); - if (initialValueKind === "binary") { - return rowEditBinaryValue(value); - } if (initialValueKind === "null" && value === "") { return null; } @@ -5842,11 +4597,7 @@ function collectRowFormValues(state) { if (control.dataset.useSqliteDefault === "1") { return; } - if ( - control.dataset.omitIfBlank === "1" && - control.value === "" && - rowEditControlValueKind(control) !== "binary" - ) { + if (control.dataset.omitIfBlank === "1" && control.value === "") { return; } if ( @@ -5880,16 +4631,6 @@ function rowEditDialogHasChanges(state) { if (!state || !state.hasLoaded || state.isLoading) { return false; } - if (state.bulkInsertInserted) { - return false; - } - if ( - state.mode === "insert" && - state.bulkInsertTextarea && - state.bulkInsertTextarea.value.trim() - ) { - return true; - } var fields = state.fields.querySelectorAll(".row-edit-field"); for (var i = 0; i < fields.length; i += 1) { var fieldApi = fields[i]._datasetteColumnFormField; @@ -5923,831 +4664,6 @@ function closeRowEditDialogIfConfirmed(state) { return true; } -function setRowInsertDialogTitle(state) { - var insertData = tableInsertData() || {}; - var title = rowEditIsMultipleInsert(state) - ? "Insert multiple rows" - : "Insert row"; - setRowDialogTitle( - state.title, - insertData.tableName ? title + " into " + insertData.tableName : title, - ); -} - -function showMultipleRowInsert(state) { - if (!state || state.mode !== "insert" || state.isSaving) { - return; - } - state.insertMode = "multiple"; - if (state.bulkInsertPreviewReady || state.bulkInsertInserted) { - resetBulkInsertPreview(state); - } - clearRowEditDialogError(state); - setRowInsertDialogTitle(state); - syncBulkInsertConflictUi(state); - syncBulkInsertTextareaValidation(state); - updateRowEditDialogButtons(state); - state.bulkInsertTextarea.focus(); -} - -function showSingleRowInsert(state) { - if (!state || state.mode !== "insert" || state.isSaving) { - return; - } - state.insertMode = "single"; - clearRowEditDialogError(state); - setRowInsertDialogTitle(state); - updateRowEditDialogButtons(state); - if (!focusFirstRowEditControl(state, { skipReadonly: true })) { - state.saveButton.focus(); - } -} - -function bulkInsertConflictMode(state) { - if (!state || !state.bulkInsertHasPrimaryKeyColumns) { - return "insert"; - } - return state.bulkInsertConflictMode || "ignore"; -} - -function syncBulkInsertConflictUi(state) { - if (!state || !state.bulkInsertConflictField) { - return; - } - var insertData = tableInsertData() || {}; - var primaryKeys = insertData.primaryKeys || []; - var hasPrimaryKeys = primaryKeys.length > 0; - var hasPrimaryKeyColumns = - hasPrimaryKeys && - bulkInsertTextIncludesPrimaryKeyColumns( - state.bulkInsertTextarea.value, - state.bulkInsertColumnDetails, - primaryKeys, - ); - state.bulkInsertHasPrimaryKeyColumns = hasPrimaryKeyColumns; - var canUpsert = hasPrimaryKeyColumns && !!state.currentUpsertUrl; - var upsertOption = state.bulkInsertConflictSelect.querySelector( - 'option[value="upsert"]', - ); - if (upsertOption) { - upsertOption.disabled = !canUpsert; - upsertOption.hidden = !canUpsert; - } - if ( - !hasPrimaryKeyColumns || - (!canUpsert && bulkInsertConflictMode(state) === "upsert") - ) { - state.bulkInsertConflictMode = hasPrimaryKeys ? "ignore" : "insert"; - state.bulkInsertConflictSelect.value = state.bulkInsertConflictMode; - } - state.bulkInsertConflictField.hidden = !hasPrimaryKeyColumns; - state.bulkInsertConflictSelect.value = bulkInsertConflictMode(state); - var helpText = ""; - if (bulkInsertConflictMode(state) === "upsert") { - helpText = - "Rows with existing primary keys will be updated; new primary keys will be inserted."; - } else if (bulkInsertConflictMode(state) === "ignore") { - helpText = "Rows with existing primary keys will be skipped."; - } else { - helpText = "Rows with existing primary keys will stop the import."; - } - state.bulkInsertConflictHelp.textContent = helpText; -} - -function bulkInsertSaveLabel(state) { - if (!state.bulkInsertPreviewReady) { - return "Preview rows"; - } - if (bulkInsertConflictMode(state) === "upsert") { - return "Update or insert rows"; - } - return "Insert these rows"; -} - -function readTextFile(file) { - if (file.text) { - return file.text(); - } - return new Promise(function (resolve, reject) { - var reader = new FileReader(); - reader.onload = function () { - resolve(reader.result || ""); - }; - reader.onerror = function () { - reject(reader.error); - }; - reader.readAsText(file); - }); -} - -async function loadBulkInsertTextFile(state, file) { - if (!file) { - return; - } - try { - state.bulkInsertTextarea.value = await readTextFile(file); - state.bulkInsertTextarea.dispatchEvent( - new Event("input", { bubbles: true }), - ); - state.bulkInsertTextarea.focus(); - } catch (_error) { - showRowEditDialogError(state, "Could not read that text file."); - } -} - -function bulkInsertTemplateText(state) { - return (state.bulkInsertTemplateColumns || []).join("\t"); -} - -async function copyTextToClipboard(text) { - if (navigator.clipboard && navigator.clipboard.writeText) { - await navigator.clipboard.writeText(text); - return; - } - var textarea = document.createElement("textarea"); - textarea.value = text; - textarea.setAttribute("readonly", ""); - textarea.style.position = "fixed"; - textarea.style.top = "-1000px"; - textarea.style.left = "-1000px"; - document.body.appendChild(textarea); - textarea.select(); - var copied = document.execCommand("copy"); - textarea.remove(); - if (!copied) { - throw new Error("copy failed"); - } -} - -function setBulkInsertCopyButtonReady(state) { - state.copyTemplateButton.textContent = ""; - var wideLabel = document.createElement("span"); - wideLabel.className = "row-edit-copy-template-label-wide"; - wideLabel.textContent = "Copy spreadsheet template"; - state.copyTemplateButton.appendChild(wideLabel); - var narrowLabel = document.createElement("span"); - narrowLabel.className = "row-edit-copy-template-label-narrow"; - narrowLabel.textContent = "Copy template"; - state.copyTemplateButton.appendChild(narrowLabel); -} - -function setBulkInsertCopyButtonCopied(state) { - state.copyTemplateButton.textContent = "Copied"; - clearTimeout(state.copyTemplateResetTimer); - state.copyTemplateResetTimer = setTimeout(function () { - setBulkInsertCopyButtonReady(state); - }, 1500); -} - -function resetBulkInsertPreview(state) { - state.bulkInsertPreviewRows = null; - state.bulkInsertPreviewReady = false; - state.bulkInsertInserted = false; - state.bulkInsertInsertedCount = 0; - state.bulkInsertPreview.hidden = true; - state.bulkInsertPreview.textContent = ""; - state.bulkInsertProgress.hidden = true; - state.bulkInsertProgressBar.value = 0; - state.bulkInsertProgressBar.max = 1; - state.bulkInsertProgressStatus.textContent = ""; - syncBulkInsertConflictUi(state); - syncRowEditInsertModeUi(state); -} - -function normalizeBulkInsertCell(column, value) { - if (typeof value === "undefined") { - return column.notnull ? "" : null; - } - if (value === null) { - return column.notnull ? "" : null; - } - if (value === "" && column.notnull) { - return ""; - } - if (column.value_kind === "number" && typeof value === "string") { - return valueFromRowEditText(column.name, value, "number"); - } - return value; -} - -function rowObjectForBulkInsert(valuesByColumn, columns) { - var row = {}; - columns.forEach(function (column) { - var hasValue = Object.prototype.hasOwnProperty.call( - valuesByColumn, - column.name, - ); - if (!hasValue) { - return; - } - row[column.name] = normalizeBulkInsertCell( - column, - valuesByColumn[column.name], - ); - }); - return row; -} - -function splitDelimitedRows(text, delimiter) { - var rows = []; - var row = []; - var cell = ""; - var inQuotes = false; - - for (var i = 0; i < text.length; i += 1) { - var character = text[i]; - if (inQuotes) { - if (character === '"') { - if (text[i + 1] === '"') { - cell += '"'; - i += 1; - } else { - inQuotes = false; - } - } else { - cell += character; - } - continue; - } - - if (character === '"') { - inQuotes = true; - } else if (character === delimiter) { - row.push(cell); - cell = ""; - } else if (character === "\n" || character === "\r") { - row.push(cell); - rows.push(row); - row = []; - cell = ""; - if (character === "\r" && text[i + 1] === "\n") { - i += 1; - } - } else { - cell += character; - } - } - - if (inQuotes) { - throw new Error("Unclosed quoted value."); - } - row.push(cell); - rows.push(row); - - while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { - rows.pop(); - } - return rows; -} - -function bulkInsertDelimitedRowIsBlank(row) { - return row.every(function (value) { - return value.trim() === ""; - }); -} - -function delimiterPreviewRows(text, delimiter) { - try { - return splitDelimitedRows(text, delimiter); - } catch (_error) { - return []; - } -} - -function splitSingleColumnRows(text) { - var rows = text.split(/\r\n|\n|\r/).map(function (line) { - return [line]; - }); - while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { - rows.pop(); - } - return rows; -} - -function detectBulkInsertDelimiter(text, columns) { - var firstLine = - text.split(/\r\n|\n|\r/).find(function (line) { - return line.trim() !== ""; - }) || ""; - var csvRows = delimiterPreviewRows(firstLine, ","); - var tsvRows = delimiterPreviewRows(firstLine, "\t"); - var csvColumns = csvRows.length ? csvRows[0].length : 0; - var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; - - if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { - return "\t"; - } - if (tsvColumns > csvColumns) { - return "\t"; - } - if (csvColumns > 1) { - return ","; - } - if (tsvColumns > 1) { - return "\t"; - } - if (columns.length === 1 || bulkInsertColumnMap(columns)[firstLine.trim()]) { - return null; - } - throw new Error("Could not detect CSV or TSV columns."); -} - -function bulkInsertColumnMap(columns) { - var map = {}; - columns.forEach(function (column) { - map[column.name] = column; - }); - return map; -} - -function bulkInsertTextIncludesPrimaryKeyColumns(text, columns, primaryKeys) { - if (!primaryKeys.length || !text.trim()) { - return false; - } - var trimmed = text.trim(); - try { - if (trimmed[0] === "[" || trimmed[0] === "{") { - return jsonBulkInsertTextIncludesPrimaryKeyColumns(trimmed, primaryKeys); - } - return delimitedBulkInsertTextIncludesPrimaryKeyColumns( - trimmed, - columns, - primaryKeys, - ); - } catch (_error) { - return false; - } -} - -function jsonBulkInsertTextIncludesPrimaryKeyColumns(text, primaryKeys) { - var rows = parseJsonObjectRows(text); - var seenKeys = {}; - rows.forEach(function (row) { - Object.keys(row).forEach(function (key) { - seenKeys[key] = true; - }); - }); - return primaryKeys.every(function (key) { - return !!seenKeys[key]; - }); -} - -function delimitedBulkInsertTextIncludesPrimaryKeyColumns( - text, - columns, - primaryKeys, -) { - var delimiter = detectBulkInsertDelimiter(text, columns); - var rows = ( - delimiter === null - ? splitSingleColumnRows(text) - : splitDelimitedRows(text, delimiter) - ).filter(function (row) { - return !bulkInsertDelimitedRowIsBlank(row); - }); - if (!rows.length) { - return false; - } - - var columnMap = bulkInsertColumnMap(columns); - var header = rows[0].map(function (value) { - return value.trim(); - }); - var headerMatches = header.filter(function (name) { - return !!columnMap[name]; - }).length; - if (headerMatches > 0) { - return primaryKeys.every(function (key) { - return header.indexOf(key) !== -1; - }); - } - - var headers = columns.map(function (column) { - return column.name; - }); - var suppliedColumnCount = rows.reduce(function (count, row) { - return Math.max(count, row.length); - }, 0); - return primaryKeys.every(function (key) { - var index = headers.indexOf(key); - return index !== -1 && index < suppliedColumnCount; - }); -} - -function bulkInsertLiveValidationShouldWait(message) { - return ( - message === "Paste rows before previewing." || - message === "No data rows found to preview." || - message.indexOf("Invalid JSON:") === 0 - ); -} - -function bulkInsertLiveValidationError(state) { - var text = state.bulkInsertTextarea.value; - if (!text.trim()) { - return null; - } - try { - parseBulkInsertRows(text, state.bulkInsertColumnDetails); - } catch (error) { - var message = error.message || "Could not preview rows."; - return bulkInsertLiveValidationShouldWait(message) ? null : message; - } - return null; -} - -function syncBulkInsertTextareaValidation(state) { - if (!rowEditIsMultipleInsert(state) || state.bulkInsertPreviewReady) { - state.bulkInsertLiveValidationError = null; - return; - } - state.bulkInsertLiveValidationError = bulkInsertLiveValidationError(state); - if (state.bulkInsertLiveValidationError) { - showRowEditDialogError(state, state.bulkInsertLiveValidationError, { - focus: false, - }); - } else { - clearRowEditDialogError(state); - } -} - -function parseJsonBulkInsertRows(text, columns) { - var parsed = parseJsonObjectRows(text); - - var columnMap = bulkInsertColumnMap(columns); - return parsed.map(function (item, index) { - if (!item || typeof item !== "object" || Array.isArray(item)) { - throw new Error("JSON row " + (index + 1) + " must be an object."); - } - Object.keys(item).forEach(function (key) { - if (!columnMap[key]) { - throw new Error( - "JSON row " + (index + 1) + " has unknown column " + key + ".", - ); - } - }); - return rowObjectForBulkInsert(item, columns); - }); -} - -function parseDelimitedBulkInsertRows(text, columns) { - var delimiter = detectBulkInsertDelimiter(text, columns); - var rows = ( - delimiter === null - ? splitSingleColumnRows(text) - : splitDelimitedRows(text, delimiter) - ).filter(function (row) { - return !bulkInsertDelimitedRowIsBlank(row); - }); - if (!rows.length) { - throw new Error("No rows found to preview."); - } - - var columnMap = bulkInsertColumnMap(columns); - var header = rows[0].map(function (value) { - return value.trim(); - }); - var headerMatches = header.filter(function (name) { - return !!columnMap[name]; - }).length; - var hasHeader = headerMatches > 0; - var dataRows = hasHeader ? rows.slice(1) : rows; - var headers = hasHeader - ? header - : columns.map(function (column) { - return column.name; - }); - var seenHeaders = {}; - - if (hasHeader) { - headers.forEach(function (name) { - if (!name) { - return; - } - if (!columnMap[name]) { - throw new Error("Unknown column " + name + " in header row."); - } - if (seenHeaders[name]) { - throw new Error("Duplicate column " + name + " in header row."); - } - seenHeaders[name] = true; - }); - } - - if (!dataRows.length) { - throw new Error("No data rows found to preview."); - } - - return dataRows.map(function (row, rowIndex) { - if (row.length > headers.length) { - throw new Error( - "Row " + - (rowIndex + 1) + - " has " + - row.length + - " values, but only " + - headers.length + - " columns were provided.", - ); - } - var valuesByColumn = {}; - row.forEach(function (value, index) { - var columnName = headers[index]; - if (columnMap[columnName]) { - valuesByColumn[columnName] = value; - } - }); - return rowObjectForBulkInsert(valuesByColumn, columns); - }); -} - -function parseBulkInsertRows(text, columns) { - var trimmed = text.trim(); - if (!trimmed) { - throw new Error("Paste rows before previewing."); - } - if (trimmed[0] === "[" || trimmed[0] === "{") { - return parseJsonBulkInsertRows(trimmed, columns); - } - return parseDelimitedBulkInsertRows(trimmed, columns); -} - -function bulkInsertPreviewValue(value) { - if (value === null) { - return "null"; - } - if (typeof value === "object") { - return JSON.stringify(value); - } - return String(value); -} - -function bulkInsertPreviewCell(column, hasValue, value) { - if (!hasValue && column.is_auto_pk) { - return { - text: "auto", - className: "row-edit-bulk-preview-auto", - }; - } - if (value === null) { - return { - text: bulkInsertPreviewValue(value), - className: "row-edit-bulk-preview-null", - }; - } - return { - text: hasValue ? bulkInsertPreviewValue(value) : "", - className: "", - }; -} - -function renderBulkInsertPreview(state, rows) { - state.bulkInsertPreview.textContent = ""; - var summary = document.createElement("p"); - summary.className = "row-edit-bulk-preview-summary"; - summary.textContent = - "Previewing " + rows.length + " row" + (rows.length === 1 ? "." : "s."); - state.bulkInsertPreview.appendChild(summary); - - var tableWrap = document.createElement("div"); - tableWrap.className = "row-edit-bulk-preview-table-wrap"; - var table = document.createElement("table"); - table.className = "row-edit-bulk-preview-table"; - var thead = document.createElement("thead"); - var headerRow = document.createElement("tr"); - state.bulkInsertColumnDetails.forEach(function (column) { - var th = document.createElement("th"); - th.scope = "col"; - th.textContent = column.name; - headerRow.appendChild(th); - }); - thead.appendChild(headerRow); - table.appendChild(thead); - - var tbody = document.createElement("tbody"); - rows.forEach(function (row) { - var tr = document.createElement("tr"); - state.bulkInsertColumnDetails.forEach(function (column) { - var td = document.createElement("td"); - var hasValue = Object.prototype.hasOwnProperty.call(row, column.name); - var value = hasValue ? row[column.name] : ""; - var cell = bulkInsertPreviewCell(column, hasValue, value); - td.textContent = cell.text; - if (cell.className) { - td.className = cell.className; - } - tr.appendChild(td); - }); - tbody.appendChild(tr); - }); - table.appendChild(tbody); - tableWrap.appendChild(table); - state.bulkInsertPreview.appendChild(tableWrap); - state.bulkInsertPreview.hidden = false; -} - -function previewBulkInsertRows(state) { - clearRowEditDialogError(state); - resetBulkInsertPreview(state); - syncBulkInsertConflictUi(state); - try { - var rows = parseBulkInsertRows( - state.bulkInsertTextarea.value, - state.bulkInsertColumnDetails, - ); - state.bulkInsertPreviewRows = rows; - state.bulkInsertPreviewReady = true; - renderBulkInsertPreview(state, rows); - updateRowEditDialogButtons(state); - } catch (error) { - showRowEditDialogError(state, error.message || "Could not preview rows."); - updateRowEditDialogButtons(state); - } -} - -function updateBulkInsertProgress(state, inserted, total) { - var words = bulkInsertProgressWords(state); - state.bulkInsertProgress.hidden = false; - state.bulkInsertProgressBar.max = total || 1; - state.bulkInsertProgressBar.value = inserted; - state.bulkInsertProgressStatus.textContent = - inserted >= total - ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." - : words.active + " " + inserted + " of " + total + " rows..."; -} - -function bulkInsertBatches(rows, batchSize) { - var batches = []; - var size = Math.max(1, batchSize || 1); - for (var index = 0; index < rows.length; index += size) { - batches.push(rows.slice(index, index + size)); - } - return batches; -} - -function animateBulkInsertProgress(state, from, to, total, duration) { - state.bulkInsertProgress.hidden = false; - state.bulkInsertProgressBar.max = total || 1; - if (duration <= 0 || !window.requestAnimationFrame) { - updateBulkInsertProgress(state, to, total); - return Promise.resolve(); - } - - return new Promise(function (resolve) { - var startTime = null; - var step = function (timestamp) { - if (startTime === null) { - startTime = timestamp; - } - var progress = Math.min((timestamp - startTime) / duration, 1); - var easedProgress = 1 - Math.pow(1 - progress, 3); - var value = from + (to - from) * easedProgress; - var displayValue = progress === 1 ? to : Math.floor(value); - var words = bulkInsertProgressWords(state); - state.bulkInsertProgressBar.value = value; - state.bulkInsertProgressStatus.textContent = - displayValue >= total - ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." - : words.active + " " + displayValue + " of " + total + " rows..."; - if (progress < 1) { - window.requestAnimationFrame(step); - } else { - updateBulkInsertProgress(state, to, total); - resolve(); - } - }; - window.requestAnimationFrame(step); - }); -} - -function bulkInsertProgressWords(state) { - var mode = bulkInsertConflictMode(state); - if (mode === "upsert") { - return { - active: "Upserting", - complete: "upserted", - }; - } - if (mode === "ignore") { - return { - active: "Processing", - complete: "processed", - }; - } - return { - active: "Inserting", - complete: "inserted", - }; -} - -function validateBulkInsertConflictRows(state, rows) { - if (bulkInsertConflictMode(state) !== "upsert") { - return null; - } - var insertData = tableInsertData() || {}; - var primaryKeys = insertData.primaryKeys || []; - for (var index = 0; index < rows.length; index += 1) { - var row = rows[index]; - var missing = primaryKeys.filter(function (key) { - return ( - !Object.prototype.hasOwnProperty.call(row, key) || - row[key] === null || - typeof row[key] === "undefined" - ); - }); - if (missing.length) { - return ( - "Row " + - (index + 1) + - " is missing primary key " + - missing.join(", ") + - ". Upsert requires primary key values for every row." - ); - } - } - return null; -} - -async function insertBulkPreviewRows(state) { - if (!state.bulkInsertPreviewRows || state.bulkInsertInserted) { - return; - } - var conflictMode = bulkInsertConflictMode(state); - var url = - conflictMode === "upsert" ? state.currentUpsertUrl : state.currentInsertUrl; - if (!url) { - showRowEditDialogError( - state, - conflictMode === "upsert" - ? "Could not find the row upsert URL." - : "Could not find the row insert URL.", - ); - return; - } - - var rows = state.bulkInsertPreviewRows; - var validationError = validateBulkInsertConflictRows(state, rows); - if (validationError) { - showRowEditDialogError(state, validationError); - return; - } - var total = rows.length; - var inserted = state.bulkInsertInsertedCount || 0; - var batches = bulkInsertBatches( - rows.slice(inserted), - state.bulkInsertMaxRows, - ); - var progressAnimationDuration = 500 / Math.max(batches.length, 1); - - clearRowEditDialogError(state); - updateBulkInsertProgress(state, inserted, total); - setRowEditDialogSaving(state, true); - try { - for (var batchIndex = 0; batchIndex < batches.length; batchIndex += 1) { - var batch = batches[batchIndex]; - var payload = { rows: batch }; - if (conflictMode === "ignore") { - payload.ignore = true; - } - var response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = null; - try { - data = await response.json(); - } catch (_error) { - data = null; - } - if (!response.ok || (data && data.ok === false)) { - throw rowMutationRequestError(response, data); - } - var previousInserted = inserted; - inserted += batch.length; - state.bulkInsertInsertedCount = inserted; - await animateBulkInsertProgress( - state, - previousInserted, - inserted, - total, - progressAnimationDuration, - ); - } - state.bulkInsertInserted = true; - state.shouldReloadOnClose = true; - state.redirectOnCloseUrl = tableBaseUrl().toString(); - updateBulkInsertProgress(state, inserted, total); - } catch (error) { - showRowEditDialogError(state, error.message || "Could not insert rows."); - } finally { - setRowEditDialogSaving(state, false); - } -} - function scheduleCloseRowEditDialogIfConfirmed(state) { // Fix for an issue in Safari where hitting Esc would show // the confirm() prompt asking if state should be discarded @@ -6846,14 +4762,6 @@ async function saveRowEditDialog(state) { if (state.isLoading || state.isSaving || !state.hasLoaded) { return; } - if (rowEditIsMultipleInsert(state)) { - if (!state.bulkInsertPreviewReady) { - previewBulkInsertRows(state); - } else if (!state.bulkInsertInserted) { - await insertBulkPreviewRows(state); - } - return; - } clearRowEditDialogError(state); setRowEditDialogSaving(state, true); @@ -7011,12 +4919,9 @@ function renderRowEditFields(state, data) { var columns = data.columns || (row ? Object.keys(row) : []); var primaryKeys = data.primary_keys || []; var columnTypes = data.column_types || {}; - var columnDetails = data.column_details || {}; - state.insertMode = "single"; destroyRowEditFields(state); columns.forEach(function (column, index) { - var columnDetail = columnDetails[column] || {}; state.fields.appendChild( createRowEditField( column, @@ -7030,9 +4935,7 @@ function renderRowEditFields(state, data) { form: state.form, manager: state.manager, mode: state.mode, - notnull: columnDetail.notnull, primaryKeyReadonly: true, - sqliteType: columnDetail.sqlite_type, }, ), ); @@ -7047,23 +4950,7 @@ function renderRowEditFields(state, data) { function renderRowInsertFields(state, data) { var columns = data.columns || []; - var bulkColumns = data.bulkColumns || columns; - state.insertMode = "single"; - state.bulkInsertColumnDetails = bulkColumns.slice(); - state.bulkInsertMaxRows = data.maxInsertRows || 100; - state.bulkInsertColumns = bulkColumns.map(function (column) { - return column.name; - }); - state.bulkInsertTemplateColumns = columns.map(function (column) { - return column.name; - }); - state.copyTemplateButton.disabled = !state.bulkInsertTemplateColumns.length; - setBulkInsertCopyButtonReady(state); - syncBulkInsertConflictUi(state); - clearTimeout(state.copyTemplateResetTimer); - state.copyTemplateResetTimer = null; - resetBulkInsertPreview(state); destroyRowEditFields(state); columns.forEach(function (column, index) { state.fields.appendChild( @@ -7153,36 +5040,7 @@ function ensureRowEditDialog(manager) {

Loading row...

-
- -
@@ -7198,27 +5056,6 @@ function ensureRowEditDialog(manager) { loading: dialog.querySelector(".row-edit-loading"), error: dialog.querySelector(".row-edit-error"), fields: dialog.querySelector(".row-edit-fields"), - bulkInsertPanel: dialog.querySelector(".row-edit-bulk"), - bulkInsertEditor: dialog.querySelector(".row-edit-bulk-editor"), - bulkInsertTextarea: dialog.querySelector(".row-edit-bulk-textarea"), - bulkInsertPreview: dialog.querySelector(".row-edit-bulk-preview"), - bulkInsertProgress: dialog.querySelector(".row-edit-bulk-progress"), - bulkInsertProgressBar: dialog.querySelector(".row-edit-bulk-progress-bar"), - bulkInsertProgressStatus: dialog.querySelector( - ".row-edit-bulk-progress-status", - ), - bulkInsertConflictField: dialog.querySelector(".row-edit-bulk-conflict"), - bulkInsertConflictSelect: dialog.querySelector( - ".row-edit-bulk-conflict-mode", - ), - bulkInsertConflictHelp: dialog.querySelector( - ".row-edit-bulk-conflict-help", - ), - copyTemplateButton: dialog.querySelector(".row-edit-copy-template"), - bulkInsertOpenFileButton: dialog.querySelector(".row-edit-bulk-open-file"), - bulkInsertFileInput: dialog.querySelector(".row-edit-bulk-file-input"), - bulkInsertLink: dialog.querySelector(".row-edit-bulk-insert"), - singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), currentButton: null, @@ -7226,25 +5063,9 @@ function ensureRowEditDialog(manager) { currentRowId: null, currentPkPath: null, currentInsertUrl: null, - currentUpsertUrl: null, currentUpdateUrl: null, currentFragmentUrl: null, mode: "edit", - insertMode: "single", - bulkInsertConflictMode: "ignore", - bulkInsertHasPrimaryKeyColumns: false, - bulkInsertLiveValidationError: null, - bulkInsertColumns: [], - bulkInsertTemplateColumns: [], - bulkInsertColumnDetails: [], - bulkInsertPreviewRows: null, - bulkInsertPreviewReady: false, - bulkInsertInserted: false, - bulkInsertInsertedCount: 0, - bulkInsertMaxRows: 100, - shouldReloadOnClose: false, - redirectOnCloseUrl: null, - copyTemplateResetTimer: null, loadId: 0, manager: manager, isLoading: false, @@ -7260,139 +5081,12 @@ function ensureRowEditDialog(manager) { }); rowEditDialogState.cancelButton.addEventListener("click", function () { - if ( - rowEditIsMultipleInsert(rowEditDialogState) && - rowEditDialogState.bulkInsertPreviewReady && - !rowEditDialogState.bulkInsertInserted && - !rowEditDialogState.isSaving - ) { - resetBulkInsertPreview(rowEditDialogState); - updateRowEditDialogButtons(rowEditDialogState); - rowEditDialogState.bulkInsertTextarea.focus(); - return; - } if (!rowEditDialogState.isSaving) { rowEditDialogState.shouldRestoreFocus = true; dialog.close(); } }); - rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { - ev.preventDefault(); - showMultipleRowInsert(rowEditDialogState); - }); - - rowEditDialogState.singleInsertLink.addEventListener("click", function (ev) { - ev.preventDefault(); - showSingleRowInsert(rowEditDialogState); - }); - - rowEditDialogState.copyTemplateButton.addEventListener( - "click", - async function () { - try { - await copyTextToClipboard(bulkInsertTemplateText(rowEditDialogState)); - clearRowEditDialogError(rowEditDialogState); - setBulkInsertCopyButtonCopied(rowEditDialogState); - } catch (_error) { - showRowEditDialogError( - rowEditDialogState, - "Could not copy the spreadsheet template.", - ); - } - }, - ); - - rowEditDialogState.bulkInsertOpenFileButton.addEventListener( - "click", - function () { - rowEditDialogState.bulkInsertFileInput.click(); - }, - ); - - rowEditDialogState.bulkInsertFileInput.addEventListener( - "change", - async function (ev) { - var files = ev.target.files; - await loadBulkInsertTextFile( - rowEditDialogState, - files && files.length ? files[0] : null, - ); - ev.target.value = ""; - }, - ); - - rowEditDialogState.bulkInsertTextarea.addEventListener( - "dragenter", - function (ev) { - ev.preventDefault(); - rowEditDialogState.bulkInsertTextarea.classList.add( - "row-edit-bulk-drop-target", - ); - }, - ); - - rowEditDialogState.bulkInsertTextarea.addEventListener( - "dragover", - function (ev) { - ev.preventDefault(); - rowEditDialogState.bulkInsertTextarea.classList.add( - "row-edit-bulk-drop-target", - ); - }, - ); - - rowEditDialogState.bulkInsertTextarea.addEventListener( - "dragleave", - function () { - rowEditDialogState.bulkInsertTextarea.classList.remove( - "row-edit-bulk-drop-target", - ); - }, - ); - - rowEditDialogState.bulkInsertTextarea.addEventListener( - "drop", - async function (ev) { - ev.preventDefault(); - rowEditDialogState.bulkInsertTextarea.classList.remove( - "row-edit-bulk-drop-target", - ); - var files = ev.dataTransfer && ev.dataTransfer.files; - if (!files || !files.length) { - return; - } - await loadBulkInsertTextFile(rowEditDialogState, files[0]); - }, - ); - - rowEditDialogState.bulkInsertTextarea.addEventListener( - "dragend", - function () { - rowEditDialogState.bulkInsertTextarea.classList.remove( - "row-edit-bulk-drop-target", - ); - }, - ); - - rowEditDialogState.bulkInsertTextarea.addEventListener("input", function () { - resetBulkInsertPreview(rowEditDialogState); - syncBulkInsertTextareaValidation(rowEditDialogState); - updateRowEditDialogButtons(rowEditDialogState); - }); - - rowEditDialogState.bulkInsertConflictSelect.addEventListener( - "change", - function () { - rowEditDialogState.bulkInsertConflictMode = - rowEditDialogState.bulkInsertConflictSelect.value; - syncBulkInsertConflictUi(rowEditDialogState); - resetBulkInsertPreview(rowEditDialogState); - syncBulkInsertTextareaValidation(rowEditDialogState); - updateRowEditDialogButtons(rowEditDialogState); - }, - ); - dialog.addEventListener("click", function (ev) { if (ev.target === dialog) { closeRowEditDialogIfConfirmed(rowEditDialogState); @@ -7414,17 +5108,8 @@ function ensureRowEditDialog(manager) { dialog.addEventListener("close", function () { var state = rowEditDialogState; - var shouldReloadOnClose = state.shouldReloadOnClose; - var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; state.isClosePending = false; - state.bulkInsertLiveValidationError = null; - state.shouldReloadOnClose = false; - state.redirectOnCloseUrl = null; - clearTimeout(state.copyTemplateResetTimer); - state.copyTemplateResetTimer = null; - setBulkInsertCopyButtonReady(state); - resetBulkInsertPreview(state); clearRowEditDialogError(state); state.hasLoaded = false; destroyRowEditFields(state); @@ -7437,13 +5122,6 @@ function ensureRowEditDialog(manager) { ) { state.currentButton.focus(); } - if (shouldReloadOnClose) { - if (redirectOnCloseUrl) { - location.href = redirectOnCloseUrl; - } else { - location.reload(); - } - } }); return rowEditDialogState; @@ -7466,10 +5144,8 @@ async function openRowEditDialog(button, manager) { state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); state.currentInsertUrl = null; - state.currentUpsertUrl = null; state.currentUpdateUrl = rowUpdateUrl(row); state.currentFragmentUrl = rowFragmentUrl(row); - state.insertMode = "single"; if (state.currentUpdateUrl) { state.form.action = new URL( state.currentUpdateUrl, @@ -7495,7 +5171,6 @@ async function openRowEditDialog(button, manager) { ); state.summary.hidden = true; state.summary.textContent = ""; - syncRowEditInsertModeUi(state); if (!state.dialog.open) { state.dialog.showModal(); @@ -7544,16 +5219,8 @@ function openRowInsertDialog(button, manager) { state.currentRowId = null; state.currentPkPath = null; state.currentInsertUrl = tableInsertUrl(); - state.currentUpsertUrl = tableUpsertUrl(); state.currentUpdateUrl = null; state.currentFragmentUrl = null; - state.insertMode = "single"; - state.bulkInsertConflictMode = "ignore"; - state.bulkInsertLiveValidationError = null; - state.bulkInsertTextarea.value = ""; - state.shouldReloadOnClose = false; - state.redirectOnCloseUrl = null; - resetBulkInsertPreview(state); state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; @@ -7571,10 +5238,14 @@ function openRowInsertDialog(button, manager) { setRowEditDialogLoading(state, false); destroyRowEditFields(state); state.dialog.removeAttribute("aria-describedby"); - setRowInsertDialogTitle(state); + setRowDialogTitle( + state.title, + insertData.tableName + ? "Insert row into " + insertData.tableName + : "Insert row", + ); state.summary.hidden = true; state.summary.textContent = ""; - syncRowEditInsertModeUi(state); if (!state.dialog.open) { state.dialog.showModal(); diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index db3c6548..a6123daa 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,9 +1,8 @@ from __future__ import annotations -import json -from collections.abc import Iterable from dataclasses import dataclass -from typing import Any +import json +from typing import Any, Iterable from .utils import tilde_encode, urlsafe_components @@ -63,6 +62,7 @@ def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: "description_html": query.description_html, "hide_sql": query.hide_sql, "fragment": query.fragment, + "params": list(query.parameters), "parameters": list(query.parameters), "is_write": query.is_write, "is_private": query.is_private, @@ -84,6 +84,7 @@ def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: return { "queries": [stored_query_to_dict(query) for query in page.queries], "next": page.next, + "has_more": page.has_more, "limit": page.limit, } @@ -387,7 +388,7 @@ async def count_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = f"%{q}%" + params["query_search"] = "%{}%".format(q) if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) @@ -463,7 +464,7 @@ async def list_queries( except ValueError: components = [] if database is None and len(components) == 3: - where_clauses.append(f""" + where_clauses.append(""" ( q.database_name > :cursor_database OR ( @@ -477,12 +478,12 @@ async def list_queries( ) ) ) - """) + """.format(sort_key_sql=sort_key_sql)) params["cursor_database"] = components[0] params["cursor_sort_key"] = components[1] params["cursor_name"] = components[2] elif database is not None and len(components) == 2: - where_clauses.append(f""" + where_clauses.append(""" ( {sort_key_sql} > :cursor_sort_key OR ( @@ -490,7 +491,7 @@ async def list_queries( AND q.name > :cursor_name ) ) - """) + """.format(sort_key_sql=sort_key_sql)) params["cursor_sort_key"] = components[0] params["cursor_name"] = components[1] @@ -503,7 +504,7 @@ async def list_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = f"%{q}%" + params["query_search"] = "%{}%".format(q) if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html index 21a2ea8f..53a824f1 100644 --- a/datasette/templates/_permission_ui_styles.html +++ b/datasette/templates/_permission_ui_styles.html @@ -6,20 +6,8 @@ padding: 1.5em; margin-bottom: 2em; } -.permission-form form { - max-width: 60rem; -} -.permission-form-grid { - display: grid; - gap: 1.5rem; - grid-template-columns: repeat(2, minmax(0, 1fr)); -} -.permission-form-result { - margin-top: 1rem; - max-width: 60rem; -} .form-section { - margin-bottom: 1.25em; + margin-bottom: 1em; } .form-section label { display: block; @@ -27,51 +15,22 @@ font-weight: bold; } .form-section input[type="text"], -.form-section input[type="number"], -.form-section select, -.permission-textarea { - background-color: #fff; - border: 1px solid #aaa; - border-radius: 4px; - box-sizing: border-box; - box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08); - color: #222; - font-family: inherit; - font-size: 1rem; - line-height: 1.4; - max-width: none; - width: 100%; -} -.form-section input[type="text"] { - height: 3rem; - padding: 0.6rem 0.75rem; -} -.form-section input[type="number"] { - height: 3rem; - max-width: 7rem; - padding: 0.6rem 0.75rem; -} .form-section select { - height: 3rem; - padding: 0.6rem 0.75rem; -} -.permission-textarea { - font-family: monospace; - min-height: 12rem; - padding: 0.75rem; - resize: vertical; + width: 100%; + max-width: 500px; + padding: 0.5em; + box-sizing: border-box; + border: 1px solid #ccc; + border-radius: 3px; } .form-section input[type="text"]:focus, -.form-section input[type="number"]:focus, -.form-section select:focus, -.permission-textarea:focus { +.form-section select:focus { + outline: 2px solid #0066cc; border-color: #0066cc; - box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18); - outline: none; } .form-section small { display: block; - margin-top: 0.45em; + margin-top: 0.3em; color: #666; } .form-actions { @@ -183,9 +142,4 @@ text-align: center; color: #666; } -@media only screen and (max-width: 576px) { - .permission-form-grid { - grid-template-columns: minmax(0, 1fr); - } -} diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html index 8e0f486e..d7203c1e 100644 --- a/datasette/templates/_permissions_debug_tabs.html +++ b/datasette/templates/_permissions_debug_tabs.html @@ -44,10 +44,10 @@ diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index fda4032c..1ecc92df 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -3,11 +3,29 @@ {% block title %}Debug allow rules{% endblock %} {% block extra_head %} -{% include "_permission_ui_styles.html" %} {% endblock %} @@ -20,28 +38,24 @@ p.message-warning {

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

-
-
-
-
- - -
-
- - -
-
-
- -
-
+
+
+

+ +
+
+

+ +
+
+ +
+
- {% if error %}

{{ error }}

{% endif %} +{% if error %}

{{ error }}

{% endif %} - {% if result == "True" %}

Result: allow

{% endif %} +{% if result == "True" %}

Result: allow

{% endif %} - {% if result == "False" %}

Result: deny

{% endif %} -
+{% if result == "False" %}

Result: deny

{% endif %} {% endblock %} diff --git a/datasette/templates/debug_actions.html b/datasette/templates/debug_actions.html index c9dccaaa..0ef7b329 100644 --- a/datasette/templates/debug_actions.html +++ b/datasette/templates/debug_actions.html @@ -9,7 +9,7 @@ {% include "_permissions_debug_tabs.html" %}

- This Datasette instance has registered {{ data.actions|length }} action{{ data.actions|length != 1 and "s" or "" }}. + This Datasette instance has registered {{ data|length }} action{{ data|length != 1 and "s" or "" }}. Actions are used by the permission system to control access to different features.

@@ -26,7 +26,7 @@ - {% for action in data.actions %} + {% for action in data %} {{ action.name }} {% if action.abbr %}{{ action.abbr }}{% endif %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 80249d9c..4f8106b8 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -49,7 +49,7 @@
- + Number of results per page (max 200)
@@ -88,7 +88,7 @@ const hasDebugPermission = {{ 'true' if has_debug_permission else 'false' }}; (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('_page'); + const page = params.get('page'); if (action) { fetchResults(page ? parseInt(page) : 1); } @@ -102,14 +102,14 @@ async function fetchResults(page = 1) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== '_size' && key !== '_page') { + if (value && key !== 'page_size') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('_page', page.toString()); - params.append('_size', pageSize); + params.append('page', page.toString()); + params.append('page_size', pageSize); try { const response = await fetch('{{ urls.path("-/allowed.json") }}?' + params.toString(), { diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index b9fc636a..3b229a25 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Explain a permission decision{% endblock %} +{% block title %}Permission Check{% endblock %} {% block extra_head %} @@ -13,35 +13,29 @@ border-radius: 5px; } #output.allowed { - background-color: #f3fbf4; + background-color: #e8f5e9; border: 2px solid #4caf50; } #output.denied { - background-color: #fff7f7; + background-color: #ffebee; border: 2px solid #f44336; } #output h2 { margin-top: 0; } -#output h3 { - margin-bottom: 0.5em; -} -#output .result-badge, -.effect-badge, -.rule-status { +#output .result-badge { display: inline-block; - padding: 0.2em 0.5em; + padding: 0.3em 0.8em; border-radius: 3px; font-weight: bold; + font-size: 1.1em; } -#output .allowed-badge, -.effect-allow { - background-color: #2e7d32; +#output .allowed-badge { + background-color: #4caf50; color: white; } -#output .denied-badge, -.effect-deny { - background-color: #c62828; +#output .denied-badge { + background-color: #f44336; color: white; } .details-section { @@ -54,130 +48,70 @@ .details-section dd { margin-left: 1em; } -.explanation-section { - background: rgba(255, 255, 255, 0.75); - border: 1px solid #ddd; - border-radius: 4px; - margin-top: 1em; - padding: 0 1em 1em; -} -.rules-table { - border-collapse: collapse; - width: 100%; -} -.rules-table th, -.rules-table td { - border-bottom: 1px solid #ddd; - padding: 0.5em; - text-align: left; - vertical-align: top; -} -.rule-status { - background: #e8f5e9; - color: #1b5e20; -} -.rule-ignored { - background: #eee; - color: #555; - font-weight: normal; -} -.requirement-allowed { - color: #1b5e20; -} -.requirement-denied { - color: #b71c1c; -} -@media only screen and (max-width: 576px) { - .rules-table, - .rules-table tbody, - .rules-table tr, - .rules-table td { - display: block; - } - .rules-table thead { - display: none; - } - .rules-table td::before { - content: attr(data-label) ": "; - font-weight: bold; - } -} {% endblock %} {% block content %} -

Explain a permission decision

+

Permission check

{% set current_tab = "check" %} {% include "_permissions_debug_tabs.html" %} -

Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

+

Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

+ +{% if request.actor %} +

Current actor: {{ request.actor.get("id", "anonymous") }}

+{% else %} +

Current actor: anonymous (not logged in)

+{% endif %}
-
+
- - - Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. -
- -
- + - The operation to evaluate + The permission action to check
-
- +
+ - The database or other parent resource + For database-level permissions, specify the database name
-
- - - The table, query or other child resource +
+ + + For table-level permissions, specify the table name (requires parent)
- +
+ {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 8b0cbbcf..4410a677 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Permission activity{% endblock %} +{% block title %}Debug permissions{% endblock %} {% block extra_head %} {% include "_permission_ui_styles.html" %} @@ -20,45 +20,60 @@ .check-action, .check-when, .check-result { font-size: 1.3em; } +textarea { + height: 10em; + width: 95%; + box-sizing: border-box; + padding: 0.5em; + border: 2px dotted black; +} +.two-col { + display: inline-block; + width: 48%; +} +.two-col label { + width: 48%; +} +@media only screen and (max-width: 576px) { + .two-col { + width: 100%; + } +} {% endblock %} {% block content %} -

Permission activity

+

Permission playground

{% set current_tab = "permissions" %} {% include "_permissions_debug_tabs.html" %} -

Raw simulator

- -

This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

+

This tool lets you simulate an actor and a permission check for that actor.

-
-
-
- - -
+
+
+ +
-
-
- - -
-
- - -
-
- - -
+
+
+
+ + +
+
+ + +
+
+ +
@@ -110,7 +125,7 @@ debugPost.addEventListener('submit', function(ev) { }); -

Recent permission checks

+

Recent permissions checks

{% if filter != "all" %}All{% else %}All{% endif %}, diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html index 233c0e94..aafa755d 100644 --- a/datasette/templates/debug_rules.html +++ b/datasette/templates/debug_rules.html @@ -37,7 +37,7 @@

- + Number of results per page (max 200)
@@ -75,7 +75,7 @@ const submitBtn = document.getElementById('submit-btn'); (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('_page'); + const page = params.get('page'); if (action) { fetchResults(page ? parseInt(page) : 1); } @@ -89,14 +89,14 @@ async function fetchResults(page = 1) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== '_size' && key !== '_page') { + if (value && key !== 'page_size') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('_page', page.toString()); - params.append('_size', pageSize); + params.append('page', page.toString()); + params.append('page_size', pageSize); try { const response = await fetch('{{ urls.path("-/rules.json") }}?' + params.toString(), { diff --git a/datasette/tokens.py b/datasette/tokens.py index 79f840d2..38a55529 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -10,7 +10,7 @@ from __future__ import annotations import dataclasses import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import itsdangerous @@ -18,21 +18,6 @@ if TYPE_CHECKING: from datasette.app import Datasette -class TokenInvalid(Exception): - """ - Raised by a TokenHandler when a token it recognizes is invalid - - for example a bad signature, malformed payload or expired token. - - Datasette responds to this with an HTTP 401 error. Handlers should - return None instead for tokens they do not recognize at all, so that - other registered handlers get a chance to verify them. - """ - - def __init__(self, message="Invalid token"): - self.message = message - super().__init__(message) - - @dataclasses.dataclass class TokenRestrictions: """ @@ -50,24 +35,24 @@ class TokenRestrictions: database: dict[str, list[str]] = dataclasses.field(default_factory=dict) resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict) - def allow_all(self, action: str) -> TokenRestrictions: + def allow_all(self, action: str) -> "TokenRestrictions": """Allow an action across all databases and resources.""" self.all.append(action) return self - def allow_database(self, database: str, action: str) -> TokenRestrictions: + def allow_database(self, database: str, action: str) -> "TokenRestrictions": """Allow an action on a specific database.""" self.database.setdefault(database, []).append(action) return self def allow_resource( self, database: str, resource: str, action: str - ) -> TokenRestrictions: + ) -> "TokenRestrictions": """Allow an action on a specific resource within a database.""" self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self - def abbreviated(self, datasette: Datasette) -> dict | None: + def abbreviated(self, datasette: "Datasette") -> Optional[dict]: """ Return the abbreviated ``_r`` dictionary shape for this set of restrictions, using action abbreviations registered with ``datasette``. @@ -112,23 +97,19 @@ class TokenHandler: async def create_token( self, - datasette: Datasette, + datasette: "Datasette", actor_id: str, *, - expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + expires_after: Optional[int] = None, + restrictions: Optional[TokenRestrictions] = None, ) -> str: """Create and return a token string for the given actor.""" raise NotImplementedError - async def verify_token(self, datasette: Datasette, token: str) -> dict | None: + async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: """ - Verify a token and return an actor dict. - - Return None if this handler does not recognize the token at all, - so other handlers can try it. Raise TokenInvalid if the token is - recognized but invalid (bad signature, malformed, expired) - the - request will fail with a 401 error. + Verify a token and return an actor dict, or None if this handler + does not recognize the token. """ raise NotImplementedError @@ -142,11 +123,11 @@ class SignedTokenHandler(TokenHandler): async def create_token( self, - datasette: Datasette, + datasette: "Datasette", actor_id: str, *, - expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + expires_after: Optional[int] = None, + restrictions: Optional[TokenRestrictions] = None, ) -> str: if not datasette.setting("allow_signed_tokens"): raise ValueError( @@ -163,35 +144,32 @@ class SignedTokenHandler(TokenHandler): token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) - async def verify_token(self, datasette: Datasette, token: str) -> dict | None: + async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: prefix = "dstok_" - if not token.startswith(prefix): - # Not one of our tokens - leave it for other handlers + if not datasette.setting("allow_signed_tokens"): return None - if not datasette.setting("allow_signed_tokens"): - raise TokenInvalid( - "Signed tokens are not enabled for this Datasette instance" - ) - max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") + if not token.startswith(prefix): + return None + raw = token[len(prefix) :] try: decoded = datasette.unsign(raw, namespace="token") except itsdangerous.BadSignature: - raise TokenInvalid("Invalid token signature") + return None if "t" not in decoded: - raise TokenInvalid("Invalid token: no timestamp") + return None created = decoded["t"] if not isinstance(created, int): - raise TokenInvalid("Invalid token: invalid timestamp") + return None duration = decoded.get("d") if duration is not None and not isinstance(duration, int): - raise TokenInvalid("Invalid token: invalid duration") + return None if (duration is None and max_signed_tokens_ttl) or ( duration is not None @@ -200,8 +178,9 @@ class SignedTokenHandler(TokenHandler): ): duration = max_signed_tokens_ttl - if duration and time.time() - created > duration: - raise TokenInvalid("Token has expired") + if duration: + if time.time() - created > duration: + return None actor = {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/tracer.py b/datasette/tracer.py index 1fbda6f9..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,11 +1,10 @@ import asyncio -import json -import time -import traceback from contextlib import contextmanager from contextvars import ContextVar - from markupsafe import escape +import time +import json +import traceback tracers = {} @@ -133,17 +132,17 @@ class AsgiTracer: "num_traces": len(traces), "traces": traces, } - content_type = next( - ( + try: + content_type = [ v.decode("utf8") for k, v in response_headers if k.lower() == b"content-type" - ), - "", - ) + ][0] + except IndexError: + content_type = "" if "text/html" in content_type and b"" in accumulated_body: extra = escape(json.dumps(trace_info, indent=2)) - extra_html = f"
{extra}
".encode() + extra_html = f"
{extra}
".encode("utf8") accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): data = json.loads(accumulated_body.decode("utf8")) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index f8da20f3..16b3d42b 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,7 +1,6 @@ +from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib -from .utils import PrefixedUrlString, path_with_format, tilde_encode - class Urls: def __init__(self, ds): @@ -9,7 +8,8 @@ class Urls: def path(self, path, format=None): if not isinstance(path, PrefixedUrlString): - path = path.removeprefix("/") + if path.startswith("/"): + path = path[1:] path = self.ds.setting("base_url") + path if format is not None: path = path_with_format(path=path, format=format) @@ -56,7 +56,6 @@ class Urls: return PrefixedUrlString(path) def row_blob(self, database, table, row_path, column): - return ( - self.table(database, table) - + f"/{row_path}.blob?_blob_column={urllib.parse.quote_plus(column)}" + return self.table(database, table) + "/{}.blob?_blob_column={}".format( + row_path, urllib.parse.quote_plus(column) ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index eb46d549..b4ede953 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,31 +1,28 @@ import asyncio -import base64 -import binascii +from contextlib import contextmanager +import aiofiles +import click +from collections import OrderedDict, namedtuple, Counter import copy import dataclasses +import base64 import hashlib import inspect import json -import os -import re -import secrets -import shlex -import shutil -import tempfile -import time -import types -import typing -import urllib -from collections import Counter, OrderedDict, namedtuple -from collections.abc import Iterable -from contextlib import contextmanager - -import aiofiles -import click import markupsafe import mergedeep +import os +import re +import shlex +import tempfile +import typing +import time +import types +import secrets +import shutil +from typing import Iterable, List, Tuple +import urllib import yaml - from .shutil_backport import copytree from .sqlite import sqlite3, supports_table_xinfo @@ -38,7 +35,7 @@ if typing.TYPE_CHECKING: class PaginatedResources: """Paginated results from allowed_resources query.""" - resources: list["Resource"] + resources: List["Resource"] next: str | None # Keyset token for next page (None if no more results) _datasette: typing.Any = dataclasses.field(default=None, repr=False) _action: str = dataclasses.field(default=None, repr=False) @@ -85,132 +82,22 @@ class PaginatedResources: # From https://www.sqlite.org/lang_keywords.html -reserved_words = { - "abort", - "action", - "add", - "after", - "all", - "alter", - "analyze", - "and", - "as", - "asc", - "attach", - "autoincrement", - "before", - "begin", - "between", - "by", - "cascade", - "case", - "cast", - "check", - "collate", - "column", - "commit", - "conflict", - "constraint", - "create", - "cross", - "current_date", - "current_time", - "current_timestamp", - "database", - "default", - "deferrable", - "deferred", - "delete", - "desc", - "detach", - "distinct", - "drop", - "each", - "else", - "end", - "escape", - "except", - "exclusive", - "exists", - "explain", - "fail", - "for", - "foreign", - "from", - "full", - "glob", - "group", - "having", - "if", - "ignore", - "immediate", - "in", - "index", - "indexed", - "initially", - "inner", - "insert", - "instead", - "intersect", - "into", - "is", - "isnull", - "join", - "key", - "left", - "like", - "limit", - "match", - "natural", - "no", - "not", - "notnull", - "null", - "of", - "offset", - "on", - "or", - "order", - "outer", - "plan", - "pragma", - "primary", - "query", - "raise", - "recursive", - "references", - "regexp", - "reindex", - "release", - "rename", - "replace", - "restrict", - "right", - "rollback", - "row", - "savepoint", - "select", - "set", - "table", - "temp", - "temporary", - "then", - "to", - "transaction", - "trigger", - "union", - "unique", - "update", - "using", - "vacuum", - "values", - "view", - "virtual", - "when", - "where", - "with", - "without", -} +reserved_words = set( + ( + "abort action add after all alter analyze and as asc attach autoincrement " + "before begin between by cascade case cast check collate column commit " + "conflict constraint create cross current_date current_time " + "current_timestamp database default deferrable deferred delete desc detach " + "distinct drop each else end escape except exclusive exists explain fail " + "for foreign from full glob group having if ignore immediate in index " + "indexed initially inner insert instead intersect into is isnull join key " + "left like limit match natural no not notnull null of offset on or order " + "outer plan pragma primary query raise recursive references regexp reindex " + "release rename replace restrict right rollback row savepoint select set " + "table temp temporary then to transaction trigger union unique update using " + "vacuum values view virtual when where with without" + ).split() +) APT_GET_DOCKERFILE_EXTRAS = r""" RUN apt-get update && \ @@ -270,7 +157,7 @@ functions_marked_as_documented = [] def documented(fn=None, *, label=None): def decorate(fn): - fn._datasette_docs_label = label or f"internals_utils_{fn.__name__}" + fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) functions_marked_as_documented.append(fn) return fn @@ -337,71 +224,24 @@ def compound_keys_after_sql(pks, start_index=0): return "({})".format("\n or\n".join(or_clauses)) -@documented class CustomJSONEncoder(json.JSONEncoder): - """ - The CustomJSONEncoder class handles serialization for objects commonly used by Datasette, - including SQLite cursors and binary blobs. Datasette uses it internally to serve .json endpoints, - and plugins that return JSON can use it to match Datasette's own handling. - - Built-in types (text, numbers, lists, etc) are encoded the same as Python's built-in ``json`` module. - - - ``sqlite3.Row`` becomes a tuple - - ``sqlite3.Cursor`` becomes a list - - Binary blobs are encoded as an object, with the actual data base64-encoded, - like so: :: - - { - "$base64": True, - "encoded": ..., - } - - Example: https://latest.datasette.io/fixtures/binary_data.json - """ - def default(self, obj): if isinstance(obj, sqlite3.Row): return tuple(obj) if isinstance(obj, sqlite3.Cursor): return list(obj) if isinstance(obj, bytes): - return { - "$base64": True, - "encoded": base64.b64encode(obj).decode("latin1"), - } + # Does it encode to utf8? + try: + return obj.decode("utf8") + except UnicodeDecodeError: + return { + "$base64": True, + "encoded": base64.b64encode(obj).decode("latin1"), + } return json.JSONEncoder.default(self, obj) -class WriteJsonValueError(ValueError): - pass - - -def decode_write_json_cell(value): - if not isinstance(value, dict): - return value - keys = set(value.keys()) - if keys == {"$raw"}: - return value["$raw"] - if keys == {"$base64", "encoded"} and value.get("$base64") is True: - encoded = value["encoded"] - if not isinstance(encoded, str): - raise WriteJsonValueError("$base64 encoded value must be a string") - try: - return base64.b64decode(encoded, validate=True) - except binascii.Error as ex: - raise WriteJsonValueError("Invalid $base64 encoded value") from ex - return value - - -def decode_write_json_row(row): - return {key: decode_write_json_cell(value) for key, value in row.items()} - - -def decode_write_json_rows(rows): - return [decode_write_json_row(row) for row in rows] - - @contextmanager def sqlite_timelimit(conn, ms): deadline = time.perf_counter() + (ms / 1000) @@ -472,7 +312,7 @@ disallawed_sql_res = [ ( re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"), "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( - ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) + ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) ), ) ] @@ -646,7 +486,10 @@ CMD {cmd}""".format( else "" ), environment_variables="\n".join( - [f"ENV {key} '{value}'" for key, value in environment_variables.items()] + [ + "ENV {} '{}'".format(key, value) + for key, value in environment_variables.items() + ] ), install_from=" ".join(install), files=" ".join(files), @@ -745,11 +588,11 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() fks = [] for info in infos: if info is not None: - id, seq, table_name, from_, to_, _on_update, _on_delete, _match = info + id, seq, table_name, from_, to_, on_update, on_delete, match = info fks.append( { "column": from_, @@ -850,7 +693,7 @@ def detect_json1(conn=None): try: conn.execute("SELECT json('{}')") return True - except sqlite3.Error: + except Exception: return False finally: if close_conn: @@ -930,7 +773,9 @@ def is_url(value): if not value.startswith("http://") and not value.startswith("https://"): return False # Any whitespace at all is invalid - return not whitespace_re.search(value) + if whitespace_re.search(value): + return False + return True css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$") @@ -983,9 +828,7 @@ def module_from_path(path, name): mod.__file__ = path with open(path, "r") as file: code = compile(file.read(), path, "exec", dont_inherit=True) - # Executing the file is the whole point - this is how --plugins-dir loads - # plugins and how metadata/config .py files are evaluated - exec(code, mod.__dict__) # noqa: S102 + exec(code, mod.__dict__) return mod @@ -1142,7 +985,9 @@ def escape_fts(query): query += '"' bits = _escape_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) + return " ".join( + '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits + ) class MultiParams: @@ -1154,7 +999,7 @@ class MultiParams: data[key], (list, tuple) ), "dictionary data should be a dictionary of key => [list]" self._data = data - elif isinstance(data, (list, tuple)): + elif isinstance(data, list) or isinstance(data, tuple): new_data = {} for item in data: assert ( @@ -1244,7 +1089,9 @@ def _gather_arguments(fn, kwargs): for parameter in parameters: if parameter not in kwargs: raise TypeError( - f"{fn} requires parameters {tuple(parameters)}, missing: {set(parameters) - set(kwargs.keys())}" + "{} requires parameters {}, missing: {}".format( + fn, tuple(parameters), set(parameters) - set(kwargs.keys()) + ) ) call_with.append(kwargs[parameter]) return call_with @@ -1313,9 +1160,9 @@ def resolve_env_secrets(config, environ): """Create copy that recursively replaces {"$env": "NAME"} with values from environ""" if isinstance(config, dict): if list(config.keys()) == ["$env"]: - return environ.get(next(iter(config.values()))) + return environ.get(list(config.values())[0]) elif list(config.keys()) == ["$file"]: - with open(next(iter(config.values()))) as fp: + with open(list(config.values())[0]) as fp: return fp.read() else: return { @@ -1393,38 +1240,29 @@ class StartupError(Exception): pass -# Comments and string literals, matched in a single pass so that whichever -# construct starts first "wins" - this ensures a comment marker inside a string -# literal (or a quote inside a comment) does not confuse the parameter scan. -_comments_and_strings_re = re.compile( - r""" - --[^\n]* # single line comment - | /\*.*?(?:\*/|\Z) # multi line comment, possibly to end-of-input - | '(?:''|[^'])*' # single quoted string ('' escapes a quote) - | "(?:""|[^"])*" # double quoted identifier ("" escapes a quote) - | \[(?:[^\]])*\] # square-bracket quoted identifier - | `(?:``|[^`])*` # backtick quoted identifier - """, - re.DOTALL | re.VERBOSE, -) +_single_line_comment_re = re.compile(r"--.*") +_multi_line_comment_re = re.compile(r"/\*.*?\*/", re.DOTALL) +_single_quote_re = re.compile(r"'(?:''|[^'])*'") +_double_quote_re = re.compile(r'"(?:\"\"|[^"])*"') _named_param_re = re.compile(r":(\w+)") @documented -def named_parameters(sql: str) -> list[str]: +def named_parameters(sql: str) -> List[str]: """ Given a SQL statement, return a list of named parameters that are used in the statement e.g. for ``select * from foo where id=:id`` this would return ``["id"]`` """ - # Strip comments and string literals first so that any ":name" sequences - # inside them are not mistaken for named parameters - sql = _comments_and_strings_re.sub("", sql) + sql = _single_line_comment_re.sub("", sql) + sql = _multi_line_comment_re.sub("", sql) + sql = _single_quote_re.sub("", sql) + sql = _double_quote_re.sub("", sql) # Extract parameters from what is left return _named_param_re.findall(sql) -async def derive_named_parameters(db: "Database", sql: str) -> list[str]: +async def derive_named_parameters(db: "Database", sql: str) -> List[str]: """ This undocumented but stable method exists for backwards compatibility with plugins that were using it before it switched to named_parameters() @@ -1432,54 +1270,6 @@ async def derive_named_parameters(db: "Database", sql: str) -> list[str]: return named_parameters(sql) -def parse_size_limit(value, default, maximum, name="_size"): - """ - Parse a page-size parameter using the same semantics as the table - view's ?_size=: blank means default, "max" means maximum, integers - must be 0 or greater and no larger than maximum. Raises ValueError - with a message suitable for a 400 response. - """ - if value in (None, ""): - return default - if value == "max": - return maximum - try: - size = int(value) - if size < 0: - raise ValueError - except ValueError: - raise ValueError(f"{name} must be a positive integer") - if size > maximum: - raise ValueError(f"{name} must be <= {maximum}") - return size - - -UNSTABLE_API_MESSAGE = ( - "This API is not part of Datasette's stable interface and may change at any time" -) - - -def error_body(messages, status): - """ - The canonical JSON error body used by every Datasette JSON error response: - - {"ok": False, "error": "...", "errors": ["...", ...], "status": 400} - - "error" is all of the messages joined with "; ", "errors" is the full - list, "status" matches the HTTP status code. Callers may add extra - context keys to the returned dictionary but must not remove these four. - """ - if isinstance(messages, str): - messages = [messages] - messages = [str(message) for message in messages] - return { - "ok": False, - "error": "; ".join(messages), - "errors": messages, - "status": status, - } - - def add_cors_headers(headers): headers["Access-Control-Allow-Origin"] = "*" headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" @@ -1508,7 +1298,7 @@ class TildeEncoder(dict): elif b == _space: res = "+" else: - res = f"~{b:02X}" + res = "~{:02X}".format(b) self[b] = res return res @@ -1603,7 +1393,7 @@ def _combine(base: dict, update: dict) -> dict: return base -def pairs_to_nested_config(pairs: list[tuple[str, typing.Any]]) -> dict: +def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: """ Parse a list of key-value pairs into a nested dictionary. """ @@ -1618,7 +1408,7 @@ def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm method = getattr(pm.hook, name, None) - assert method is not None, f"No hook found for {name}" + assert method is not None, "No hook found for {}".format(name) async def inner(): html_bits = [] @@ -1642,7 +1432,7 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins_and_allow(source: dict, destination: dict) -> tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: """ Move 'plugins' and 'allow' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index d767e391..c7137e6b 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -252,62 +252,88 @@ async def _build_single_action_sql( ] ) - # Continue with the cascading logic. - # Aggregate the RULES by cascade level (small), rather than grouping - # base x rules (which scales with the number of resources). - def _agg(select_key, where, group_by): - parts = [ - f" SELECT {select_key}", - " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,", - " json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons", - f" FROM all_rules WHERE {where}", - ] - if group_by: - parts.append(f" GROUP BY {group_by}") - return parts - + # Continue with the cascading logic query_parts.extend( - ["child_agg AS ("] - + _agg( - "parent, child,", - "parent IS NOT NULL AND child IS NOT NULL", - "parent, child", - ) - + ["),", "parent_agg AS ("] - + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") - + ["),", "global_agg AS ("] - + _agg("", "parent IS NULL AND child IS NULL", None) - + ["),"] + [ + "child_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", + " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", + " FROM base b", + " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child", + " GROUP BY b.parent, b.child", + "),", + "parent_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", + " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", + " FROM base b", + " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + "global_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", + " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", + " FROM base b", + " LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + ] ) # Add anonymous decision logic if needed if include_is_private: - - def _anon_agg(select_key, where, group_by): - parts = [ - f" SELECT {select_key}", - " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow", - f" FROM anon_rules WHERE {where}", - ] - if group_by: - parts.append(f" GROUP BY {group_by}") - return parts - query_parts.extend( - ["anon_child_agg AS ("] - + _anon_agg( - "parent, child,", - "parent IS NOT NULL AND child IS NOT NULL", - "parent, child", - ) - + ["),", "anon_parent_agg AS ("] - + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") - + ["),", "anon_global_agg AS ("] - + _anon_agg("", "parent IS NULL AND child IS NULL", None) - + ["),"] + [ + "anon_child_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", + " FROM base b", + " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child", + " GROUP BY b.parent, b.child", + "),", + "anon_parent_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", + " FROM base b", + " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + "anon_global_lvl AS (", + " SELECT b.parent, b.child,", + " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", + " FROM base b", + " LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL", + " GROUP BY b.parent, b.child", + "),", + "anon_decisions AS (", + " SELECT", + " b.parent, b.child,", + " CASE", + " WHEN acl.any_deny = 1 THEN 0", + " WHEN acl.any_allow = 1 THEN 1", + " WHEN apl.any_deny = 1 THEN 0", + " WHEN apl.any_allow = 1 THEN 1", + " WHEN agl.any_deny = 1 THEN 0", + " WHEN agl.any_allow = 1 THEN 1", + " ELSE 0", + " END AS anon_is_allowed", + " FROM base b", + " JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))", + " JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))", + " JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))", + "),", + ] ) # Final decisions @@ -316,28 +342,31 @@ async def _build_single_action_sql( "decisions AS (", " SELECT", " b.parent, b.child,", - " -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level", + " -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level", " -- Priority order:", - " -- 1. Child-level deny 2. Child-level allow", - " -- 3. Parent-level deny 4. Parent-level allow", - " -- 5. Global-level deny 6. Global-level allow", + " -- 1. Child-level deny (most specific, blocks access)", + " -- 2. Child-level allow (most specific, grants access)", + " -- 3. Parent-level deny (intermediate, blocks access)", + " -- 4. Parent-level allow (intermediate, grants access)", + " -- 5. Global-level deny (least specific, blocks access)", + " -- 6. Global-level allow (least specific, grants access)", " -- 7. Default deny (no rules match)", " CASE", - " WHEN ca.any_deny = 1 THEN 0", - " WHEN ca.any_allow = 1 THEN 1", - " WHEN pa.any_deny = 1 THEN 0", - " WHEN pa.any_allow = 1 THEN 1", - " WHEN ga.any_deny = 1 THEN 0", - " WHEN ga.any_allow = 1 THEN 1", + " WHEN cl.any_deny = 1 THEN 0", + " WHEN cl.any_allow = 1 THEN 1", + " WHEN pl.any_deny = 1 THEN 0", + " WHEN pl.any_allow = 1 THEN 1", + " WHEN gl.any_deny = 1 THEN 0", + " WHEN gl.any_allow = 1 THEN 1", " ELSE 0", " END AS is_allowed,", " CASE", - " WHEN ca.any_deny = 1 THEN ca.deny_reasons", - " WHEN ca.any_allow = 1 THEN ca.allow_reasons", - " WHEN pa.any_deny = 1 THEN pa.deny_reasons", - " WHEN pa.any_allow = 1 THEN pa.allow_reasons", - " WHEN ga.any_deny = 1 THEN ga.deny_reasons", - " WHEN ga.any_allow = 1 THEN ga.allow_reasons", + " WHEN cl.any_deny = 1 THEN cl.deny_reasons", + " WHEN cl.any_allow = 1 THEN cl.allow_reasons", + " WHEN pl.any_deny = 1 THEN pl.deny_reasons", + " WHEN pl.any_allow = 1 THEN pl.allow_reasons", + " WHEN gl.any_deny = 1 THEN gl.deny_reasons", + " WHEN gl.any_allow = 1 THEN gl.allow_reasons", " ELSE '[]'", " END AS reason", ] @@ -345,34 +374,21 @@ async def _build_single_action_sql( if include_is_private: query_parts.append( - " , CASE WHEN (" - "CASE" - " WHEN aca.any_deny = 1 THEN 0" - " WHEN aca.any_allow = 1 THEN 1" - " WHEN apa.any_deny = 1 THEN 0" - " WHEN apa.any_allow = 1 THEN 1" - " WHEN aga.any_deny = 1 THEN 0" - " WHEN aga.any_allow = 1 THEN 1" - " ELSE 0 END" - ") = 0 THEN 1 ELSE 0 END AS is_private" + " , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private" ) query_parts.extend( [ " FROM base b", - " LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child", - " LEFT JOIN parent_agg pa ON pa.parent = b.parent", - " CROSS JOIN global_agg ga", + " JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))", + " JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))", + " JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))", ] ) if include_is_private: - query_parts.extend( - [ - " LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child", - " LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent", - " CROSS JOIN anon_global_agg aga", - ] + query_parts.append( + " JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))" ) query_parts.append(")") @@ -384,28 +400,8 @@ async def _build_single_action_sql( restriction_intersect = "\nINTERSECT\n".join( f"SELECT * FROM ({sql})" for sql in restriction_sqls ) - # Decompose by NULL-pattern so the final filter can use pure-equality - # EXISTS lookups (satisfiable via automatic indexes) instead of a - # correlated OR-scan over the whole list. query_parts.extend( - [ - ",", - "restriction_list AS (", - f" {restriction_intersect}", - "),", - "restriction_exact AS (", - " SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL", - "),", - "restriction_parent_any AS (", - " SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL", - "),", - "restriction_child_any AS (", - " SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL", - "),", - "restriction_all AS (", - " SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1", - ")", - ] + [",", "restriction_list AS (", f" {restriction_intersect}", ")"] ) # Final SELECT @@ -420,11 +416,10 @@ async def _build_single_action_sql( # Add restriction filter if there are restrictions if restriction_sqls: query_parts.append(""" - AND ( - EXISTS (SELECT 1 FROM restriction_all) - OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent) - OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child) - OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child) + AND EXISTS ( + SELECT 1 FROM restriction_list r + WHERE (r.parent = decisions.parent OR r.parent IS NULL) + AND (r.child = decisions.child OR r.child IS NULL) )""") # Add parent filter if specified @@ -678,239 +673,3 @@ async def check_permission_for_resource( child=child, ) return results[action] - - -async def explain_permission_for_resource( - *, - datasette: "Datasette", - actor: dict | None, - action: str, - parent: str | None, - child: str | None, -) -> dict: - """Explain a permission decision for one action and resource. - - This is intended for Datasette's permission debugging tools. It uses the - same ``permission_resources_sql`` hook results and the same resolution - rules as :func:`check_permissions_for_actions`, but also returns the - matching rules, actor restriction results and ``also_requires`` chain. - - The returned dictionary is part of Datasette's unstable debugging API. - """ - - action_obj = datasette.actions.get(action) - if action_obj is None: - raise ValueError(f"Unknown action: {action}") - - explanation = await _explain_single_action( - datasette=datasette, - actor=actor, - action=action, - parent=parent, - child=child, - ) - - required_actions = [] - if action_obj.also_requires: - required = await explain_permission_for_resource( - datasette=datasette, - actor=actor, - action=action_obj.also_requires, - parent=parent, - child=child, - ) - required_actions.append(required) - - explanation["required_actions"] = required_actions - explanation["allowed"] = bool( - explanation["rule_allowed"] - and explanation["restriction_allowed"] - and all(required["allowed"] for required in required_actions) - ) - explanation["summary"] = _permission_explanation_summary(explanation) - return explanation - - -async def _explain_single_action( - *, - datasette: "Datasette", - actor: dict | None, - action: str, - parent: str | None, - child: str | None, -) -> dict: - """Return matching rules and restrictions for a single action.""" - from datasette.utils.permissions import SKIP_PERMISSION_CHECKS - - permission_sqls = await gather_permission_sql_from_hooks( - datasette=datasette, - actor=actor, - action=action, - ) - - if permission_sqls is SKIP_PERMISSION_CHECKS: - return { - "action": action, - "rule_allowed": True, - "restriction_allowed": True, - "winning_scope": "global", - "matched_rules": [ - { - "scope": "global", - "effect": "allow", - "source": "skip_permission_checks", - "reason": "Permission checks were explicitly skipped", - "decisive": True, - "ignored_because": None, - } - ], - "restrictions": [], - } - - db = datasette.get_internal_database() - matched_rules = [] - restrictions = [] - - for permission_sql in permission_sqls: - params = dict(permission_sql.params or {}) - parent_param = _unused_parameter_name(params, "_explain_parent") - params[parent_param] = parent - child_param = _unused_parameter_name(params, "_explain_child") - params[child_param] = child - - if permission_sql.sql: - rows = await db.execute( - f""" - SELECT parent, child, allow, reason - FROM ({permission_sql.sql}) AS permission_rules - WHERE (parent IS NULL OR parent = :{parent_param}) - AND (child IS NULL OR child = :{child_param}) - """, - params, - ) - for row in rows: - specificity = ( - 2 - if row["child"] is not None - else 1 if row["parent"] is not None else 0 - ) - matched_rules.append( - { - "scope": ("resource", "parent", "global")[2 - specificity], - "effect": "allow" if row["allow"] else "deny", - "source": permission_sql.source, - "reason": row["reason"], - "_specificity": specificity, - } - ) - - if permission_sql.restriction_sql: - restriction_row = ( - await db.execute( - f""" - SELECT EXISTS( - SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules - WHERE (parent IS NULL OR parent = :{parent_param}) - AND (child IS NULL OR child = :{child_param}) - ) AS resource_is_in_allowlist - """, - params, - ) - ).first() - restriction_allowed = bool(restriction_row[0]) - restrictions.append( - { - "source": permission_sql.source, - "allowed": restriction_allowed, - "reason": params.get("deny") - or ( - "Resource is included in this restriction allowlist" - if restriction_allowed - else "Resource is not included in this restriction allowlist" - ), - } - ) - - matched_rules.sort( - key=lambda rule: ( - -rule["_specificity"], - 0 if rule["effect"] == "deny" else 1, - rule["source"] or "", - rule["reason"] or "", - ) - ) - - if matched_rules: - winning_specificity = matched_rules[0]["_specificity"] - winning_rules = [ - rule - for rule in matched_rules - if rule["_specificity"] == winning_specificity - ] - rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules) - winning_scope = winning_rules[0]["scope"] - else: - winning_specificity = None - rule_allowed = False - winning_scope = None - - for rule in matched_rules: - specificity = rule.pop("_specificity") - if specificity != winning_specificity: - rule["decisive"] = False - rule["ignored_because"] = "A more specific rule matched" - elif not rule_allowed and rule["effect"] == "allow": - rule["decisive"] = False - rule["ignored_because"] = "A deny rule matched at the same scope" - else: - rule["decisive"] = True - rule["ignored_because"] = None - - return { - "action": action, - "rule_allowed": rule_allowed, - "restriction_allowed": all( - restriction["allowed"] for restriction in restrictions - ), - "winning_scope": winning_scope, - "matched_rules": matched_rules, - "restrictions": restrictions, - } - - -def _unused_parameter_name(params: dict, preferred: str) -> str: - """Return a SQL parameter name that is not already in ``params``.""" - candidate = preferred - suffix = 2 - while candidate in params: - candidate = f"{preferred}_{suffix}" - suffix += 1 - return candidate - - -def _permission_explanation_summary(explanation: dict) -> str: - denied_requirement = next( - ( - required - for required in explanation["required_actions"] - if not required["allowed"] - ), - None, - ) - if denied_requirement: - return ( - f"Denied because {explanation['action']} also requires " - f"{denied_requirement['action']}, which was denied." - ) - if not explanation["matched_rules"]: - return "Denied because no permission rule matched this actor and resource." - if not explanation["rule_allowed"]: - return ( - f"Denied by a {explanation['winning_scope']}-level rule. " - "Deny rules take precedence over allow rules at the same scope." - ) - if not explanation["restriction_allowed"]: - return ( - "Denied because the resource is not included in the actor's restrictions." - ) - return f"Allowed by the matching {explanation['winning_scope']}-level rule." diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 2614ad02..e1631b10 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,30 +1,28 @@ -import asyncio import json -import re -from http.cookies import Morsel, SimpleCookie -from mimetypes import guess_type -from pathlib import Path -from urllib.parse import parse_qs, parse_qsl, urlunparse - -import aiofiles -import aiofiles.os - -from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file +from typing import Optional +from datasette.utils import MultiParams, calculate_etag, sha256_file from datasette.utils.multipart import ( - DEFAULT_MAX_FIELD_SIZE, - DEFAULT_MAX_FIELDS, + parse_form_data, + MultipartParseError, + FormData, DEFAULT_MAX_FILE_SIZE, + DEFAULT_MAX_REQUEST_SIZE, + DEFAULT_MAX_FIELDS, DEFAULT_MAX_FILES, + DEFAULT_MAX_PARTS, + DEFAULT_MAX_FIELD_SIZE, DEFAULT_MAX_MEMORY_FILE_SIZE, DEFAULT_MAX_PART_HEADER_BYTES, DEFAULT_MAX_PART_HEADER_LINES, - DEFAULT_MAX_PARTS, - DEFAULT_MAX_REQUEST_SIZE, DEFAULT_MIN_FREE_DISK_BYTES, - FormData, - MultipartParseError, - parse_form_data, ) +from mimetypes import guess_type +from urllib.parse import parse_qs, urlunparse, parse_qsl +from pathlib import Path +from http.cookies import SimpleCookie, Morsel +import aiofiles +import aiofiles.os +import re # Workaround for adding samesite support to pre 3.8 python Morsel._reserved["samesite"] = "SameSite" @@ -69,28 +67,16 @@ class BadRequest(Base400): status = 400 -class PayloadTooLarge(Base400): - status = 413 - - SAMESITE_VALUES = ("strict", "lax", "none") -# Bodies read fully into memory (post_body/post_vars/json) are capped at this -# size unless the max_post_body_bytes setting says otherwise. Kept deliberately -# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk, -# while these bodies are held in RAM and json.loads() can multiply their -# footprint several times over. -DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB - class Request: - def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES): + def __init__(self, scope, receive): self.scope = scope self.receive = receive - self.max_post_body_bytes = max_post_body_bytes def __repr__(self): - return f'' + return ''.format(self.method, self.url) @property def method(self): @@ -155,43 +141,15 @@ class Request: def actor(self): return self.scope.get("actor", None) - async def post_body(self, max_bytes=None): - """ - Read the request body fully into memory. - - The body is capped at max_bytes - or self.max_post_body_bytes - (default 2MB, set from the max_post_body_bytes setting for requests - created by Datasette) if max_bytes is not provided. Pass max_bytes=0 - to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded - - oversized bodies are rejected as soon as the limit is passed, without - buffering the rest. - """ - if max_bytes is None: - max_bytes = self.max_post_body_bytes - too_large = PayloadTooLarge( - f"Request body exceeded maximum size of {max_bytes} bytes" - ) - if max_bytes: - # Reject early if the client declares an oversized body - try: - if int(self.headers.get("content-length", "")) > max_bytes: - raise too_large - except ValueError: - # Missing or malformed - the streaming check below still applies - pass - chunks = [] - received = 0 + async def post_body(self): + body = b"" more_body = True while more_body: message = await self.receive() assert message["type"] == "http.request", message - chunk = message.get("body", b"") - received += len(chunk) - if max_bytes and received > max_bytes: - raise too_large - chunks.append(chunk) + body += message.get("body", b"") more_body = message.get("more_body", False) - return b"".join(chunks) + return body async def post_vars(self): body = await self.post_body() @@ -208,7 +166,7 @@ class Request: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -301,24 +259,12 @@ class AsgiLifespan: while True: message = await receive() if message["type"] == "lifespan.startup": - try: - for fn in self.on_startup: - await fn() - except Exception as e: # noqa: BLE001 - await send( - {"type": "lifespan.startup.failed", "message": str(e)} - ) - return + for fn in self.on_startup: + await fn() await send({"type": "lifespan.startup.complete"}) elif message["type"] == "lifespan.shutdown": - try: - for fn in self.on_shutdown: - await fn() - except Exception as e: # noqa: BLE001 - await send( - {"type": "lifespan.shutdown.failed", "message": str(e)} - ) - return + for fn in self.on_shutdown: + await fn() await send({"type": "lifespan.shutdown.complete"}) return else: @@ -543,9 +489,9 @@ class Response: httponly=False, samesite="lax", ): - assert ( - samesite in SAMESITE_VALUES - ), f"samesite should be one of {SAMESITE_VALUES}" + assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format( + SAMESITE_VALUES + ) cookie = SimpleCookie() cookie[key] = value for prop_name, prop_value in ( @@ -589,18 +535,6 @@ class Response: content_type="application/json; charset=utf-8", ) - @classmethod - def error(cls, messages, status=400, headers=None): - """ - A JSON error response using Datasette's standard error format. - - messages can be a single string or a list of strings. For errors - that should content-negotiate between JSON and HTML, raise - Forbidden, NotFound, BadRequest or DatasetteError instead and let - Datasette's error handling hooks build the response. - """ - return cls.json(error_body(messages, status), status=status, headers=headers) - @classmethod def redirect(cls, path, status=302, headers=None): headers = headers or {} @@ -637,23 +571,10 @@ class AsgiRunOnFirstRequest: self.asgi = asgi self.on_startup = on_startup self._started = False - # Guards against concurrent early requests interleaving with startup: - # without this, several requests could all observe `_started is - # False` and proceed before any of them finish running the hooks. - self._lock = asyncio.Lock() async def __call__(self, scope, receive, send): - # Leave "lifespan" scope events alone - this shim only exists as a - # fallback for hosts that never send them. It wraps AsgiLifespan, so - # if it ran on_startup here too, a startup exception would escape - # before AsgiLifespan's own try/except got a chance to turn it into - # a lifespan.startup.failed message. - if scope["type"] != "lifespan" and not self._started: - async with self._lock: - # Re-check: another request may have finished startup while - # we were waiting for the lock. - if not self._started: - for hook in self.on_startup: - await hook() - self._started = True + if not self._started: + self._started = True + for hook in self.on_startup: + await hook() return await self.asgi(scope, receive, send) diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index 0469d7a8..c4b64908 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -13,7 +13,7 @@ Originally shared here: https://www.djangosnippets.org/snippets/1431/ """ -class BaseConverter: +class BaseConverter(object): decimal_digits = "0123456789" def __init__(self, digits): diff --git a/datasette/utils/check_callable.py b/datasette/utils/check_callable.py index e21a769b..a0997d20 100644 --- a/datasette/utils/check_callable.py +++ b/datasette/utils/check_callable.py @@ -1,6 +1,6 @@ import inspect import types -from typing import Any, NamedTuple +from typing import NamedTuple, Any class CallableStatus(NamedTuple): @@ -19,7 +19,7 @@ def check_callable(obj: Any) -> CallableStatus: if isinstance(obj, types.FunctionType): return CallableStatus(True, inspect.iscoroutinefunction(obj)) - if callable(obj): + if hasattr(obj, "__call__"): return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__)) - assert False, f"obj {obj!r} is somehow callable with no __call__ method" + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 0ddeb847..bf172667 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,30 +1,9 @@ import textwrap +from datasette.utils import table_column_details -from sqlite_utils import Database as SQLiteUtilsDatabase -from sqlite_utils import Migrations -from datasette.utils import escape_sqlite, table_column_details - -INTERNAL_DB_SCHEMA_TABLES = { - "catalog_databases", - "catalog_tables", - "catalog_views", - "catalog_columns", - "catalog_indexes", - "catalog_foreign_keys", - "metadata_instance", - "metadata_databases", - "metadata_resources", - "metadata_columns", - "column_types", - "queries", -} - -INTERNAL_DB_SCHEMA_INDEXES = { - "queries_owner_idx", -} - -INTERNAL_DB_SCHEMA_SQL = textwrap.dedent(""" +async def init_internal_db(db): + create_tables_sql = textwrap.dedent(""" CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, @@ -88,101 +67,99 @@ INTERNAL_DB_SCHEMA_SQL = textwrap.dedent(""" FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name) ); - - CREATE TABLE IF NOT EXISTS metadata_instance ( - key text, - value text, - unique(key) - ); - - CREATE TABLE IF NOT EXISTS metadata_databases ( - database_name text, - key text, - value text, - unique(database_name, key) - ); - - CREATE TABLE IF NOT EXISTS metadata_resources ( - database_name text, - resource_name text, - key text, - value text, - unique(database_name, resource_name, key) - ); - - CREATE TABLE IF NOT EXISTS metadata_columns ( - database_name text, - resource_name text, - column_name text, - key text, - value text, - unique(database_name, resource_name, column_name, key) - ); - - CREATE TABLE IF NOT EXISTS column_types ( - database_name TEXT NOT NULL, - resource_name TEXT NOT NULL, - column_name TEXT NOT NULL, - column_type TEXT NOT NULL, - config TEXT, - PRIMARY KEY (database_name, resource_name, column_name) - ); - - CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) - ); - - CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); """).strip() + await db.execute_write_script(create_tables_sql) + await initialize_metadata_tables(db) -internal_migrations = Migrations("datasette_internal") +async def initialize_metadata_tables(db): + await db.execute_write_script(textwrap.dedent(""" + CREATE TABLE IF NOT EXISTS metadata_instance ( + key text, + value text, + unique(key) + ); + + CREATE TABLE IF NOT EXISTS metadata_databases ( + database_name text, + key text, + value text, + unique(database_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_resources ( + database_name text, + resource_name text, + key text, + value text, + unique(database_name, resource_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_columns ( + database_name text, + resource_name text, + column_name text, + key text, + value text, + unique(database_name, resource_name, column_name, key) + ); + + CREATE TABLE IF NOT EXISTS column_types ( + database_name TEXT NOT NULL, + resource_name TEXT NOT NULL, + column_name TEXT NOT NULL, + column_type TEXT NOT NULL, + config TEXT, + PRIMARY KEY (database_name, resource_name, column_name) + ); + + CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name) + ); + + CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); + """)) -def _internal_schema_exists(db): - table_names = set(db.table_names()) - if not INTERNAL_DB_SCHEMA_TABLES.issubset(table_names): - return False - index_names = { - row[0] - for row in db.execute("select name from sqlite_master where type = 'index'") - } - return INTERNAL_DB_SCHEMA_INDEXES.issubset(index_names) - - -@internal_migrations(name="0001_initial") -def initial_internal_schema(db): - if _internal_schema_exists(db): - return - db.executescript(INTERNAL_DB_SCHEMA_SQL) - - -async def init_internal_db(db): - def apply_migrations(conn): - internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False)) - - await db.execute_write_fn(apply_migrations, transaction=False) - - -async def populate_schema_tables(internal_db, db, schema_version): +async def populate_schema_tables(internal_db, db): database_name = db.name + def delete_everything(conn): + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_views WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) + + await internal_db.execute_write_fn(delete_everything) + tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows @@ -207,30 +184,25 @@ async def populate_schema_tables(internal_db, db, schema_version): columns = table_column_details(conn, table_name) columns_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **column._asdict(), } for column in columns ) foreign_keys = conn.execute( - f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" + f"PRAGMA foreign_key_list([{table_name}])" ).fetchall() foreign_keys_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **dict(foreign_key), } for foreign_key in foreign_keys ) - indexes = conn.execute( - f"PRAGMA index_list({escape_sqlite(table_name)})" - ).fetchall() + indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() indexes_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **dict(index), } for index in indexes @@ -251,76 +223,47 @@ async def populate_schema_tables(internal_db, db, schema_version): indexes_to_insert, ) = await db.execute_fn(collect_info) - def replace_catalog(conn): - # Delete child rows before their catalog_tables parents so this also - # works if a prepare_connection plugin enables foreign key enforcement. - for table in ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - ): - conn.execute( - f"DELETE FROM {table} WHERE database_name = ?", - [database_name], - ) - conn.execute( - """ - INSERT OR REPLACE INTO catalog_databases ( - database_name, path, is_memory, schema_version - ) VALUES (?, ?, ?, ?) - """, - [ - database_name, - str(db.path) if db.path is not None else None, - db.is_memory, - schema_version, - ], + await internal_db.execute_write_many( + """ + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_views (database_name, view_name, rootpage, sql) + values (?, ?, ?, ?) + """, + views_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden ) - conn.executemany( - """ - INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - tables_to_insert, + """, + columns_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match ) - conn.executemany( - """ - INSERT INTO catalog_views (database_name, view_name, rootpage, sql) - values (?, ?, ?, ?) - """, - views_to_insert, + """, + foreign_keys_to_insert, + ) + await internal_db.execute_write_many( + """ + INSERT INTO catalog_indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial ) - conn.executemany( - """ - INSERT INTO catalog_columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden - ) - """, - columns_to_insert, - ) - conn.executemany( - """ - INSERT INTO catalog_foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match - ) - """, - foreign_keys_to_insert, - ) - conn.executemany( - """ - INSERT INTO catalog_indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial - ) - """, - indexes_to_insert, - ) - - await internal_db.execute_write_fn(replace_catalog) + """, + indexes_to_insert, + ) diff --git a/datasette/utils/multipart.py b/datasette/utils/multipart.py index 13289b1c..cfa77486 100644 --- a/datasette/utils/multipart.py +++ b/datasette/utils/multipart.py @@ -11,10 +11,15 @@ Supports: import asyncio import shutil import tempfile -from collections.abc import Callable from dataclasses import dataclass, field from typing import ( Any, + Callable, + Dict, + List, + Optional, + Tuple, + Union, ) from urllib.parse import parse_qsl @@ -24,7 +29,7 @@ DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024 # 100MB DEFAULT_MAX_FIELDS = 1000 DEFAULT_MAX_FILES = 100 # If max_parts is not specified, it defaults to max_fields + max_files -DEFAULT_MAX_PARTS: int | None = None +DEFAULT_MAX_PARTS: Optional[int] = None DEFAULT_MAX_FIELD_SIZE = 100 * 1024 # 100KB DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024 # 1MB DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024 # 16KB @@ -35,6 +40,8 @@ DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024 # 50MB class MultipartParseError(Exception): """Raised when multipart parsing fails.""" + pass + @dataclass class UploadedFile: @@ -50,7 +57,7 @@ class UploadedFile: name: str filename: str - content_type: str | None + content_type: Optional[str] size: int _file: tempfile.SpooledTemporaryFile = field(repr=False) @@ -79,8 +86,7 @@ class UploadedFile: def __del__(self): try: self._file.close() - except Exception: # noqa: BLE001, S110 - # __del__ must never raise + except Exception: pass @@ -92,27 +98,27 @@ class FormData: """ def __init__(self): - self._data: list[tuple[str, str | UploadedFile]] = [] + self._data: List[Tuple[str, Union[str, UploadedFile]]] = [] - def append(self, key: str, value: str | UploadedFile) -> None: + def append(self, key: str, value: Union[str, UploadedFile]) -> None: """Add a key-value pair.""" self._data.append((key, value)) - def __getitem__(self, key: str) -> str | UploadedFile: + def __getitem__(self, key: str) -> Union[str, UploadedFile]: """Get the first value for a key.""" for k, v in self._data: if k == key: return v raise KeyError(key) - def get(self, key: str, default: Any = None) -> str | UploadedFile | None: + def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]: """Get the first value for a key, or default if not found.""" try: return self[key] except KeyError: return default - def getlist(self, key: str) -> list[str | UploadedFile]: + def getlist(self, key: str) -> List[Union[str, UploadedFile]]: """Get all values for a key.""" return [v for k, v in self._data if k == key] @@ -136,15 +142,15 @@ class FormData: """Return unique keys.""" return list(self) - def items(self) -> list[tuple[str, str | UploadedFile]]: + def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]: """Return all key-value pairs.""" return list(self._data) - def values(self) -> list[str | UploadedFile]: + def values(self) -> List[Union[str, UploadedFile]]: """Return all values.""" return [v for _, v in self._data] - def _uploaded_files(self) -> list[UploadedFile]: + def _uploaded_files(self) -> List[UploadedFile]: """Return UploadedFile instances contained in this form.""" return [v for _, v in self._data if isinstance(v, UploadedFile)] @@ -157,7 +163,7 @@ class FormData: for uploaded in self._uploaded_files(): try: uploaded.close_sync() - except Exception: # noqa: BLE001, S110 + except Exception: # Best-effort cleanup; ignore close errors pass @@ -166,7 +172,7 @@ class FormData: for uploaded in self._uploaded_files(): try: await uploaded.close() - except Exception: # noqa: BLE001, S110 + except Exception: # Best-effort cleanup; ignore close errors pass @@ -183,13 +189,13 @@ class FormData: await self.aclose() -def parse_content_disposition(header: str) -> dict[str, str | None]: +def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: """ Parse Content-Disposition header value. Returns dict with 'name', 'filename' keys (filename may be None). """ - result: dict[str, str | None] = {"name": None, "filename": None} + result: Dict[str, Optional[str]] = {"name": None, "filename": None} # Split on semicolons, handling quoted strings parts = [] @@ -232,8 +238,7 @@ def parse_content_disposition(header: str) -> dict[str, str | None]: from urllib.parse import unquote result["filename"] = unquote(encoded, encoding="utf-8") - except Exception: # noqa: BLE001, S110 - # Malformed RFC 5987 filename* - fall back to the plain filename + except Exception: pass continue @@ -245,19 +250,20 @@ def parse_content_disposition(header: str) -> dict[str, str | None]: if key == "name": result["name"] = value - # Only set filename if filename* hasn't already set it - elif key == "filename" and result["filename"] is None: - # Strip path components (security) - # Handle both Unix and Windows paths - value = value.replace("\\", "/") - if "/" in value: - value = value.rsplit("/", 1)[-1] - result["filename"] = value + elif key == "filename": + # Only set if filename* hasn't already set it + if result["filename"] is None: + # Strip path components (security) + # Handle both Unix and Windows paths + value = value.replace("\\", "/") + if "/" in value: + value = value.rsplit("/", 1)[-1] + result["filename"] = value return result -def parse_content_type(header: str) -> tuple[str, dict[str, str]]: +def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]: """ Parse Content-Type header value. @@ -301,7 +307,7 @@ class MultipartParser: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -342,12 +348,12 @@ class MultipartParser: self._tempdir = tempfile.gettempdir() # Current part state - self.current_headers: dict[str, str] = {} - self.current_file: tempfile.SpooledTemporaryFile | None = None + self.current_headers: Dict[str, str] = {} + self.current_file: Optional[tempfile.SpooledTemporaryFile] = None self.current_body = bytearray() - self.current_name: str | None = None - self.current_filename: str | None = None - self.current_content_type: str | None = None + self.current_name: Optional[str] = None + self.current_filename: Optional[str] = None + self.current_content_type: Optional[str] = None def feed(self, chunk: bytes) -> None: """Feed a chunk of data to the parser.""" @@ -448,7 +454,7 @@ class MultipartParser: # Parse header try: line_str = line.decode("utf-8", errors="replace") - except UnicodeDecodeError: + except Exception: line_str = line.decode("latin-1") if ":" in line_str: @@ -475,9 +481,7 @@ class MultipartParser: if self.file_count > self.max_files: raise MultipartParseError("Too many files") if self.handle_files: - # Outlives this method - it is filled in across parser callbacks - # and then handed to the UploadedFile the caller consumes - self.current_file = tempfile.SpooledTemporaryFile( # noqa: SIM115 + self.current_file = tempfile.SpooledTemporaryFile( max_size=self.max_memory_file_size ) else: @@ -640,7 +644,7 @@ async def parse_form_data( max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, diff --git a/datasette/utils/permissions.py b/datasette/utils/permissions.py index 5a8ee8e2..fd1e41a1 100644 --- a/datasette/utils/permissions.py +++ b/datasette/utils/permissions.py @@ -2,9 +2,8 @@ from __future__ import annotations import json +from typing import Any, Dict, Iterable, List, Sequence, Tuple import sqlite3 -from collections.abc import Iterable, Sequence -from typing import Any from datasette.permissions import PermissionSQL from datasette.plugins import pm @@ -16,7 +15,7 @@ SKIP_PERMISSION_CHECKS = object() async def gather_permission_sql_from_hooks( *, datasette, actor: dict | None, action: str -) -> list[PermissionSQL] | object: +) -> List[PermissionSQL] | object: """Collect PermissionSQL objects from the permission_resources_sql hook. Ensures that each returned PermissionSQL has a populated ``source``. @@ -35,7 +34,7 @@ async def gather_permission_sql_from_hooks( hookimpls = hook_caller.get_hookimpls() hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action)) - collected: list[PermissionSQL] = [] + collected: List[PermissionSQL] = [] actor_json = json.dumps(actor) if actor is not None else None actor_id = actor.get("id") if isinstance(actor, dict) else None @@ -72,7 +71,7 @@ def _iter_permission_sql_from_result( if isinstance(result, PermissionSQL): return [result] if isinstance(result, (list, tuple)): - collected: list[PermissionSQL] = [] + collected: List[PermissionSQL] = [] for item in result: collected.extend(_iter_permission_sql_from_result(item, action=action)) return collected @@ -91,7 +90,7 @@ def _iter_permission_sql_from_result( def build_rules_union( actor: dict | None, plugins: Sequence[PermissionSQL] -) -> tuple[str, dict[str, Any]]: +) -> Tuple[str, Dict[str, Any]]: """ Compose plugin SQL into a UNION ALL. @@ -103,10 +102,10 @@ def build_rules_union( The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent Plugin parameters should be prefixed with a unique identifier (e.g., source name). """ - parts: list[str] = [] + parts: List[str] = [] actor_json = json.dumps(actor) if actor else None actor_id = actor.get("id") if actor else None - params: dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} + params: Dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} for p in plugins: # No namespacing - just use plugin params as-is @@ -142,10 +141,10 @@ async def resolve_permissions_from_catalog( plugins: Sequence[Any], action: str, candidate_sql: str, - candidate_params: dict[str, Any] | None = None, + candidate_params: Dict[str, Any] | None = None, *, implicit_deny: bool = True, -) -> list[dict[str, Any]]: +) -> List[Dict[str, Any]]: """ Resolve permissions by embedding the provided *candidate_sql* in a CTE. @@ -169,8 +168,8 @@ async def resolve_permissions_from_catalog( - parent, child, allow, reason, source_plugin, depth - resource (rendered "/parent/child" or "/parent" or "/") """ - resolved_plugins: list[PermissionSQL] = [] - restriction_sqls: list[str] = [] + resolved_plugins: List[PermissionSQL] = [] + restriction_sqls: List[str] = [] for plugin in plugins: if callable(plugin) and not isinstance(plugin, PermissionSQL): @@ -399,11 +398,11 @@ async def resolve_permissions_with_candidates( db, actor: dict | None, plugins: Sequence[Any], - candidates: list[tuple[str, str | None]], + candidates: List[Tuple[str, str | None]], action: str, *, implicit_deny: bool = True, -) -> list[dict[str, Any]]: +) -> List[Dict[str, Any]]: """ Resolve permissions without any external candidate table by embedding the candidates as a UNION of parameterized SELECTs in a CTE. @@ -412,8 +411,8 @@ async def resolve_permissions_with_candidates( actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action """ # Build a small CTE for candidates. - cand_rows_sql: list[str] = [] - cand_params: dict[str, Any] = {} + cand_rows_sql: List[str] = [] + cand_params: Dict[str, Any] = {} for i, (parent, child) in enumerate(candidates): pkey = f"cand_p_{i}" ckey = f"cand_c_{i}" diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index d323f5d6..d1fd1bd7 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -6,7 +6,7 @@ https://github.com/python/cpython/blob/v3.8.3/LICENSE """ import os -from shutil import Error, copy, copy2, copystat +from shutil import copy, copy2, copystat, Error def _copytree( diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 334545bd..0a3a947c 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -150,6 +150,7 @@ _SQLITE_INTERNAL_SCHEMA_FUNCTIONS = { "sqlite_rename_test", "substr", } + _AUTHORIZER_ACTION_NAMES = { getattr(sqlite3, name): name for name in ( @@ -390,10 +391,6 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK - if action == sqlite3.SQLITE_RECURSIVE: - # Recursive CTE bookkeeping; table reads are reported separately. - return sqlite3.SQLITE_OK - if action == sqlite3.SQLITE_FUNCTION and arg2 is not None: record( "function", @@ -413,12 +410,12 @@ def analyze_sql_tables( database=None, table=None, sqlite_schema=sqlite_schema, - target=f"{arg1} {arg2}" if arg2 is not None else arg1, + target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, source=source, ) return sqlite3.SQLITE_OK - action_name = _AUTHORIZER_ACTION_NAMES.get(action, f"SQLITE_{action}") + action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) record( "unknown", "unknown", @@ -488,17 +485,17 @@ def analyze_sql_tables( and key.operation in {"create", "alter", "drop"} for key in operations ) - dropped_tables_and_views = { + dropped_tables = { (key.database, key.table) for key in operations - if key.operation == "drop" and key.target_type in {"table", "view"} + if key.operation == "drop" and key.target_type == "table" } def key_is_drop_table_delete(key: OperationKey) -> bool: return ( key.operation == "delete" and key.target_type == "table" - and (key.database, key.table) in dropped_tables_and_views + and (key.database, key.table) in dropped_tables ) has_user_table_access_in_schema_operation = any( @@ -521,7 +518,9 @@ def analyze_sql_tables( and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS ): return True - return bool(key_is_drop_table_delete(key)) + if key_is_drop_table_delete(key): + return True + return False def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index d3926f6f..4743ae4c 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -100,7 +100,7 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str] schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" + "select name, sql from {} where type = 'table'".format(schema_table) ).fetchall() except sqlite3.DatabaseError: return [] @@ -127,7 +127,7 @@ def _sqlite_table_type_from_schema( schema_table = _sqlite_schema_table(schema) try: row = conn.execute( - f"select type, sql from {schema_table} where name = ?", + "select type, sql from {} where name = ?".format(schema_table), (table,), ).fetchone() except sqlite3.DatabaseError: @@ -155,7 +155,7 @@ def _is_known_shadow_table( schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" + "select name, sql from {} where type = 'table'".format(schema_table) ).fetchall() except sqlite3.DatabaseError: return False @@ -174,7 +174,7 @@ def _sqlite_schema_table(schema: str | None) -> str: return "sqlite_master" if schema == "temp": return "sqlite_temp_master" - return f"{_quote_identifier(schema)}.sqlite_master" + return "{}.sqlite_master".format(_quote_identifier(schema)) def _quote_identifier(value: str) -> str: diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index a8be47bf..de7e94af 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -1,7 +1,6 @@ -import json -from urllib.parse import urlencode - from asgiref.sync import async_to_sync +from urllib.parse import urlencode +import json # These wrapper classes pre-date the introduction of # datasette.client and httpx to Datasette. They could diff --git a/datasette/version.py b/datasette/version.py index 2ec12fd2..49d270e4 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a38" +__version__ = "1.0a35" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index bac3b39e..ed7e175f 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,7 +1,7 @@ +from dataclasses import dataclass import dataclasses import types import typing -from dataclasses import dataclass @dataclass(frozen=True) @@ -74,14 +74,16 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - f"{cls.__name__}.{name} is declared with from_extra() but there is no " - "registered extra of that name" + "{}.{} is declared with from_extra() but there is no " + "registered extra of that name".format(cls.__name__, name) ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - f"{cls.__name__}.{name} is declared with from_extra() but the {name} extra is " - f"not available for scope {cls.extras_scope}" + "{}.{} is declared with from_extra() but the {} extra is " + "not available for scope {}".format( + cls.__name__, name, name, cls.extras_scope + ) ) return extra_class.description or "" diff --git a/datasette/views/base.py b/datasette/views/base.py index 48108cec..30026f4b 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -2,20 +2,20 @@ import csv import hashlib import sys +from datasette.utils.asgi import Request from datasette.utils import ( + add_cors_headers, EscapeHtmlWriter, InvalidSql, LimitedWriter, - add_cors_headers, path_from_row_pks, path_with_format, sqlite3, ) from datasette.utils.asgi import ( AsgiStream, - BadRequest, - Request, Response, + BadRequest, ) @@ -28,15 +28,12 @@ class DatasetteError(Exception): status=500, template=None, message_is_html=False, - plain_message=None, ): self.message = message self.title = title self.error_dict = error_dict or {} self.status = status self.message_is_html = message_is_html - # Plain text used for JSON error responses when message is HTML - self.plain_message = plain_message class View: @@ -52,7 +49,9 @@ class View: request.path.endswith(".json") or request.headers.get("content-type") == "application/json" ): - response = Response.error("Method not allowed", 405) + response = Response.json( + {"ok": False, "error": "Method not allowed"}, status=405 + ) else: response = Response.text("Method not allowed", status=405) return response @@ -91,7 +90,9 @@ class BaseView: request.path.endswith(".json") or request.headers.get("content-type") == "application/json" ): - response = Response.error("Method not allowed", 405) + response = Response.json( + {"ok": False, "error": "Method not allowed"}, status=405 + ) else: response = Response.text("Method not allowed", status=405) return response @@ -129,10 +130,12 @@ class BaseView: template = environment.select_template(templates) template_context = { **context, - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], + **{ + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + }, } headers = {} if self.has_json_alternate: @@ -149,7 +152,9 @@ class BaseView: template_context["alternate_url_json"] = alternate_url_json headers.update( { - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) } ) return Response.html( @@ -175,12 +180,18 @@ class BaseView: return view +def _error(messages, status=400): + return Response.json({"ok": False, "errors": messages}, status=status) + + async def stream_csv(datasette, fetch_data, request, database): kwargs = {} stream = request.args.get("_stream") # Do not calculate facets or counts: extra_parameters = [ - f"{key}=1" for key in ("_nofacet", "_nocount") if not request.args.get(key) + "{}=1".format(key) + for key in ("_nofacet", "_nocount") + if not request.args.get(key) ] if extra_parameters: # Replace request object with a new one with modified scope @@ -210,6 +221,9 @@ async def stream_csv(datasette, fetch_data, request, database): except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: raise @@ -316,9 +330,8 @@ async def stream_csv(datasette, fetch_data, request, database): else: new_row.append(cell) await writer.writerow(new_row) - except Exception as ex: # noqa: BLE001 - # Streaming CSV: report the error into the response body and stop - sys.stderr.write(f"Caught this error: {ex}\n") + except Exception as ex: + sys.stderr.write("Caught this error: {}\n".format(ex)) sys.stderr.flush() await r.write(str(ex)) return diff --git a/datasette/views/database.py b/datasette/views/database.py index f54ffd38..e02de657 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,52 +1,48 @@ +from dataclasses import asdict, dataclass, field +from urllib.parse import parse_qsl, urlencode import asyncio import hashlib import itertools import json +import markupsafe import os import textwrap -from dataclasses import asdict, dataclass, field -from urllib.parse import parse_qsl, urlencode - -import markupsafe +from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.plugins import pm from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, stored_query_to_dict +from datasette.write_sql import QueryWriteRejected from datasette.utils import ( - InvalidSql, add_cors_headers, await_me_maybe, call_with_supported_arguments, - error_body, + named_parameters as derive_named_parameters, format_bytes, - is_url, make_slot_function, + tilde_decode, + to_css_class, + validate_sql_select, + is_url, path_with_added_args, path_with_format, path_with_removed_args, sqlite3, - tilde_decode, - to_css_class, truncate_url, - validate_sql_select, + InvalidSql, ) -from datasette.utils import ( - named_parameters as derive_named_parameters, -) -from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response -from datasette.write_sql import QueryWriteRejected +from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden +from datasette.plugins import pm -from . import Context from .base import DatasetteError, View, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns -from .table_create_alter import _create_table_ui_context from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) +from .table_create_alter import _create_table_ui_context +from . import Context @dataclass @@ -103,7 +99,7 @@ class DatabaseView(View): return response if format_ not in ("html", "json"): - raise NotFound(f"Invalid format: {format_}") + raise NotFound("Invalid format: {}".format(format_)) metadata = await datasette.get_database_metadata(database) @@ -167,7 +163,7 @@ class DatabaseView(View): "label": "Create table", "description": "Create a new table in this database.", "attrs": { - "aria-label": f"Create table in {database}", + "aria-label": "Create table in {}".format(database), "data-database-action": "create-table", }, } @@ -274,7 +270,9 @@ class DatabaseView(View): view_name="database", ), headers={ - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) }, ) @@ -327,7 +325,7 @@ class DatabaseContext(Context): database_color: str = field(metadata={"help": "The color assigned to the database"}) database_page_data: dict = field( metadata={ - "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.' + "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.' } ) database_actions: callable = field( @@ -557,7 +555,7 @@ async def database_download(request, datasette): if datasette.cors: add_cors_headers(headers) if db.hash: - etag = f'"{db.hash}"' + etag = '"{}"'.format(db.hash) headers["Etag"] = etag # Has user seen this already? if_none_match = request.headers.get("if-none-match") @@ -609,7 +607,11 @@ class QueryView(View): "_json" ): return Response.json( - dict(error_body([ex.message], 403), redirect=None), + { + "ok": False, + "message": ex.message, + "redirect": None, + }, status=403, ) datasette.add_message(request, ex.message, datasette.ERROR) @@ -644,15 +646,8 @@ class QueryView(View): ok = None redirect_url = None try: - execute_write_kwargs = {"request": request} - if stored_query.is_trusted: - analysis = await db.analyze_sql(stored_query.sql, params_for_query) - if any( - operation.operation == "vacuum" for operation in analysis.operations - ): - execute_write_kwargs["transaction"] = False cursor = await db.execute_write( - stored_query.sql, params_for_query, **execute_write_kwargs + stored_query.sql, params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None @@ -665,9 +660,8 @@ class QueryView(View): ).first() if message_result: message = message_result[0] - except Exception as ex: # noqa: BLE001 - # Stored-query on_success_message_sql is user-authored - message = f"Error running on_success_message_sql: {ex}" + except Exception as ex: + message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: if stored_query.on_success_message: @@ -681,24 +675,18 @@ class QueryView(View): redirect_url = stored_query.on_success_redirect ok = True - except Exception as ex: # noqa: BLE001 - # Stored-query execution is user-authored SQL + except Exception as ex: message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR redirect_url = stored_query.on_error_redirect ok = False if should_return_json: - if ok: - return Response.json( - { - "ok": True, - "message": message, - "redirect": redirect_url, - } - ) return Response.json( - dict(error_body([message], 400), redirect=redirect_url), - status=400, + { + "ok": ok, + "message": message, + "redirect": redirect_url, + } ) else: datasette.add_message(request, message, message_type) @@ -816,23 +804,19 @@ class QueryView(View): rows = results.rows except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(f""" + textwrap.dedent("""

SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

- + - """).strip(), + """.format(markupsafe.escape(ex.sql))).strip(), title="SQL Interrupted", status=400, message_is_html=True, - plain_message=( - "SQL query took too long. The time limit is" - " controlled by the sql_time_limit_ms setting." - ), ) except sqlite3.DatabaseError as ex: query_error = str(ex) @@ -841,6 +825,8 @@ class QueryView(View): columns = [] except (sqlite3.OperationalError, InvalidSql) as ex: raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) except DatasetteError: raise @@ -862,12 +848,9 @@ class QueryView(View): return data, None, None return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers: - if not sql: - raise DatasetteError("?sql= is required", status=400) + elif format_ in datasette.renderers.keys(): data = {"ok": True, "rows": rows, "columns": columns} extras = extra_names_from_request(request) - table_extra_registry.validate_requested(extras, ExtraScope.QUERY) if extras: query_extra_context = QueryExtraContext( datasette=datasette, @@ -954,7 +937,9 @@ class QueryView(View): } headers.update( { - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) } ) metadata = await query_metadata() @@ -1035,7 +1020,9 @@ class QueryView(View): + "?" + urlencode( { - "sql": sql, + **{ + "sql": sql, + }, **named_parameter_values, } ) @@ -1137,7 +1124,7 @@ class QueryView(View): headers=headers, ) else: - assert False, f"Invalid format: {format_}" + assert False, "Invalid format: {}".format(format_) if datasette.cors: add_cors_headers(r.headers) return r @@ -1238,7 +1225,7 @@ async def display_rows(datasette, database, request, rows, columns): '<Binary: {:,} byte{}>'.format( blob_url, ( - f' title="{formatted}"' + ' title="{}"'.format(formatted) if "bytes" not in formatted else "" ), diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 3dfb810c..b7e8288e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -2,14 +2,14 @@ import re from urllib.parse import urlencode from datasette.resources import DatabaseResource -from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3 +from datasette.utils import sqlite3 from datasette.utils.asgi import Response -from .base import BaseView +from .base import BaseView, _error from .database import display_rows as display_query_rows from .query_helpers import ( - SQL_PARAMETER_FORM_PREFIX, QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -31,7 +31,15 @@ WRITE_TEMPLATE_LABELS = { "delete": "Delete rows", } WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) -CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" +CREATE_TABLE_TEMPLATE_SQL = "\n".join( + ( + "create table new_table (", + " id integer primary key,", + " name text", + " -- created text default (datetime('now'))", + ")", + ) +) def _parameter_names(columns): @@ -41,11 +49,11 @@ def _parameter_names(columns): base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) base = base.strip("_") or "value" if base[0].isdigit(): - base = f"p_{base}" + base = "p_{}".format(base) name = base index = 2 while name in seen: - name = f"{base}_{index}" + name = "{}_{}".format(base, index) index += 1 seen.add(name) names[column] = name @@ -57,7 +65,7 @@ def _quote_identifier(identifier): def _preferred_where_column(table, columns): - lower_table_id = f"{table.lower()}_id" + lower_table_id = "{}_id".format(table.lower()) return ( next((column for column in columns if column.lower() == "id"), None) or next( @@ -82,15 +90,17 @@ def _insert_template_sql(table, columns): auto_pk = _auto_incrementing_primary_key(columns) insert_columns = [column for column in column_names if column != auto_pk] if not insert_columns: - return f"insert into {_quote_identifier(table)}\ndefault values" + return "insert into {}\ndefault values".format(_quote_identifier(table)) names = _parameter_names(insert_columns) return "\n".join( ( - f"insert into {_quote_identifier(table)} (", - ",\n".join(f" {_quote_identifier(column)}" for column in insert_columns), + "insert into {} (".format(_quote_identifier(table)), + ",\n".join( + " {}".format(_quote_identifier(column)) for column in insert_columns + ), ")", "values (", - ",\n".join(f" :{names[column]}" for column in insert_columns), + ",\n".join(" :{}".format(names[column]) for column in insert_columns), ")", ) ) @@ -104,14 +114,18 @@ def _update_template_sql(table, columns): if not set_columns: return "\n".join( ( - f"update {_quote_identifier(table)}", - f"set {_quote_identifier(where_column)} = :new_{names[where_column]}", - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "update {}".format(_quote_identifier(table)), + "set {} = :new_{}".format( + _quote_identifier(where_column), names[where_column] + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) return "\n".join( ( - f"update {_quote_identifier(table)}", + "update {}".format(_quote_identifier(table)), "set " + ",\n".join( "{}{} = :{}".format( @@ -121,7 +135,9 @@ def _update_template_sql(table, columns): ) for index, column in enumerate(set_columns) ), - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) @@ -132,8 +148,10 @@ def _delete_template_sql(table, columns): where_column = _preferred_where_column(table, column_names) return "\n".join( ( - f"delete from {_quote_identifier(table)}", - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "delete from {}".format(_quote_identifier(table)), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) @@ -330,7 +348,7 @@ class ExecuteWriteView(BaseView): ) if not db.is_mutable: return _block_framing( - Response.error( + _error( ["Cannot execute write SQL because this database is immutable."], 403, ) @@ -349,10 +367,10 @@ class ExecuteWriteView(BaseView): actor=request.actor, ): return _block_framing( - Response.error(["Permission denied: need execute-write-sql"], 403) + _error(["Permission denied: need execute-write-sql"], 403) ) if not db.is_mutable: - return _block_framing(Response.error(["Database is immutable"], 403)) + return _block_framing(_error(["Database is immutable"], 403)) data = {} is_json = request.headers.get("content-type", "").startswith("application/json") @@ -366,7 +384,7 @@ class ExecuteWriteView(BaseView): ) except QueryValidationError as ex: if _wants_json(request, is_json, data): - return _block_framing(Response.error([ex.message], ex.status)) + return _block_framing(_error([ex.message], ex.status)) if ex.flash: self.ds.add_message(request, ex.message, self.ds.ERROR) return await self._render_form( @@ -387,7 +405,7 @@ class ExecuteWriteView(BaseView): except sqlite3.DatabaseError as ex: message = str(ex) if wants_json: - return _block_framing(Response.error([message], 400)) + return _block_framing(_error([message], 400)) return await self._render_form( request, db, @@ -470,18 +488,20 @@ class ExecuteWriteAnalyzeView(BaseView): actor=request.actor, ): return _block_framing( - Response.error(["Permission denied: need execute-write-sql"], 403) + _error(["Permission denied: need execute-write-sql"], 403) ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - Response.error( + _error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) ) sql = request.args.get("sql") or "" - analysis = await _execute_write_analysis_data(self.ds, db, sql, request.actor) - analysis["unstable"] = UNSTABLE_API_MESSAGE - return _block_framing(Response.json(analysis)) + return _block_framing( + Response.json( + await _execute_write_analysis_data(self.ds, db, sql, request.actor) + ) + ) diff --git a/datasette/views/index.py b/datasette/views/index.py index f73ee38a..6a9462ac 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -2,11 +2,10 @@ import json from datasette.plugins import pm from datasette.utils import ( - UNSTABLE_API_MESSAGE, - CustomJSONEncoder, add_cors_headers, await_me_maybe, make_slot_function, + CustomJSONEncoder, ) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -46,15 +45,15 @@ class IndexView(BaseView): databases = [] # Iterate over allowed databases instead of all databases - for name, allowed_db in allowed_db_dict.items(): + for name in allowed_db_dict.keys(): db = self.ds.databases[name] - database_private = allowed_db.private + database_private = allowed_db_dict[name].private # Get allowed tables/views for this database allowed_for_db = tables_by_db.get(name, {}) # Get table names from allowed set instead of db.table_names() - table_names = [child_name for child_name in allowed_for_db] + table_names = [child_name for child_name in allowed_for_db.keys()] hidden_table_names = set(await db.hidden_table_names()) @@ -99,7 +98,7 @@ class IndexView(BaseView): # We will be sorting by number of relationships, so populate that field all_foreign_keys = await db.get_all_foreign_keys() for table, foreign_keys in all_foreign_keys.items(): - if table in tables: + if table in tables.keys(): count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) tables[table]["num_relationships_for_sorting"] = count @@ -121,7 +120,8 @@ class IndexView(BaseView): # Only add views if this is less than TRUNCATE_AT if len(tables_and_views_truncated) < TRUNCATE_AT: num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated) - tables_and_views_truncated.extend(views[:num_views_to_add]) + for view in views[:num_views_to_add]: + tables_and_views_truncated.append(view) databases.append( { @@ -151,9 +151,7 @@ class IndexView(BaseView): return Response( json.dumps( { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, - "databases": databases, + "databases": {db["name"]: db for db in databases}, "metadata": await self.ds.get_instance_metadata(), }, cls=CustomJSONEncoder, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 725d9cdb..026a999f 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,19 +5,6 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, ) -from datasette.utils import ( - InvalidSql, - escape_sqlite, - parse_size_limit, - path_from_row_pks, - sqlite3, - validate_sql_select, -) -from datasette.utils import ( - named_parameters as derive_named_parameters, -) -from datasette.utils.asgi import Forbidden -from datasette.utils.sql_analysis import Operation, SQLAnalysis from datasette.write_sql import ( IgnoreWriteSqlOperation, QueryWriteRejected, @@ -25,6 +12,16 @@ from datasette.write_sql import ( decision_for_write_sql_operation, operation_is_write, ) +from datasette.utils import ( + named_parameters as derive_named_parameters, + escape_sqlite, + path_from_row_pks, + sqlite3, + validate_sql_select, + InvalidSql, +) +from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -35,6 +32,7 @@ _query_fields = { "hide_sql", "fragment", "parameters", + "params", "is_private", "on_success_message", "on_success_redirect", @@ -93,14 +91,16 @@ def _as_optional_bool(value, name): return True if lowered in {"0", "false", "f", "no", "off"}: return False - raise QueryValidationError(f"{name} must be 0 or 1") + raise QueryValidationError("{} must be 0 or 1".format(name)) -def _query_list_limit(value, default, maximum): +def _query_list_limit(value, default=50): + if value in (None, ""): + return default try: - return parse_size_limit(value, default, maximum) + return min(max(1, int(value)), 1000) except ValueError as ex: - raise QueryValidationError(str(ex)) from ex + raise QueryValidationError("_size must be an integer") from ex def _derived_query_parameters(sql): @@ -173,7 +173,7 @@ async def _json_or_form_payload(request): try: return json.loads(body or b"{}"), True except json.JSONDecodeError as e: - raise QueryValidationError(f"Invalid JSON: {e}") + raise QueryValidationError("Invalid JSON: {}".format(e)) return await request.post_vars(), False @@ -194,7 +194,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError(f"Could not analyze query: {ex}") from ex + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex is_write = _analysis_is_write(analysis) if is_write: @@ -295,7 +295,8 @@ def _coerce_execute_write_payload(data, is_json): for key, value in data.items(): if key in {"sql", "csrftoken", "_json"}: continue - key = key.removeprefix(SQL_PARAMETER_FORM_PREFIX) + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") @@ -315,7 +316,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError(f"Could not analyze query: {ex}") from ex + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex if not _analysis_is_write(analysis): raise QueryValidationError( "Use /-/query for read-only SQL; this endpoint only executes writes" @@ -497,7 +498,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): ) try: result = await db.execute( - f"select {select} from {escape_sqlite(table)} where rowid = ?", + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), [lastrowid], ) except sqlite3.DatabaseError: @@ -540,7 +541,7 @@ async def _prepare_query_create(datasette, request, db, data): raise QueryValidationError("Writable query fields require writable SQL") parameters = _coerce_query_parameters( - data.get("parameters"), + data.get("parameters", data.get("params")), derived, ) return { @@ -585,9 +586,9 @@ async def _prepare_query_update(datasette, request, db, existing: StoredQuery, u actor=request.actor, ) - if "parameters" in update: + if "parameters" in update or "params" in update: parameters = _coerce_query_parameters( - update.get("parameters"), + update.get("parameters", update.get("params")), derived, ) elif "sql" in update: diff --git a/datasette/views/row.py b/datasette/views/row.py index b1388299..129216b9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,35 +8,32 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils +from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted -from datasette.events import DeleteRowEvent, UpdateRowEvent -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.plugins import pm +from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource +from .base import BaseView, DatasetteError, _error, stream_csv from datasette.utils import ( - CustomJSONEncoder, - CustomRow, - InvalidSql, - WriteJsonValueError, add_cors_headers, await_me_maybe, call_with_supported_arguments, - decode_write_json_row, - escape_sqlite, + CustomRow, + InvalidSql, make_slot_function, path_from_row_pks, + path_with_added_args, path_with_format, path_with_removed_args, - sqlite3, to_css_class, + escape_sqlite, + sqlite3, ) -from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response - +from datasette.plugins import pm +from datasette.extras import extra_names_from_request, ExtraScope from . import Context, from_extra -from .base import BaseView, DatasetteError, stream_csv from .table import ( - _table_page_data, display_columns_and_rows, + _table_page_data, row_label_from_label_column, ) from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -188,33 +185,43 @@ class RowView(BaseView): data, extra_template_data, templates = response_or_template_contexts except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(f""" + textwrap.dedent("""

SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

- + - """).strip(), + """.format(markupsafe.escape(ex.sql))).strip(), title="SQL Interrupted", status=400, message_is_html=True, - plain_message=( - "SQL query took too long. The time limit is" - " controlled by the sql_time_limit_ms setting." - ), ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) except DatasetteError: raise end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - if format_ in self.ds.renderers: + # Special case for .jsono extension - redirect to _shape=objects + if format_ == "jsono": + return self.redirect( + request, + path_with_added_args( + request, + {"_shape": "objects"}, + path=request.path.rsplit(".jsono", 1)[0] + ".json", + ), + forward_querystring=False, + ) + + if format_ in self.ds.renderers.keys(): # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -257,7 +264,7 @@ class RowView(BaseView): if status_code is not None: response.status = status_code else: - raise NotFound(f"Invalid format: {format_}") + raise NotFound("Invalid format: {}".format(format_)) ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): @@ -372,7 +379,9 @@ class RowView(BaseView): view_name=self.name, ), headers={ - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) }, ) @@ -497,7 +506,7 @@ class RowView(BaseView): row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = f"{pk_path} {row_label}" + row_action_label = "{} {}".format(pk_path, row_label) row_action_permissions = {} if is_table and db.is_mutable: @@ -510,7 +519,7 @@ class RowView(BaseView): row_actions = [] if row_action_permissions.get("update-row"): attrs = { - "aria-label": f"Edit row {row_action_label}", + "aria-label": "Edit row {}".format(row_action_label), "data-row": row_path, "data-row-action": "edit", } @@ -526,7 +535,7 @@ class RowView(BaseView): ) if row_action_permissions.get("delete-row"): attrs = { - "aria-label": f"Delete row {row_action_label}", + "aria-label": "Delete row {}".format(row_action_label), "data-row": row_path, "data-row-action": "delete", } @@ -600,9 +609,6 @@ class RowView(BaseView): } extras = extra_names_from_request(request) - if request.url_vars.get("format"): - # Data formats reject unknown extras; HTML ignores them - table_extra_registry.validate_requested(extras, ExtraScope.ROW) # Process extras row_extra_context = RowExtraContext( @@ -676,7 +682,7 @@ class RowView(BaseView): key, ",".join(pk_values), ) - foreign_key_tables.append({**fk, "count": count, "link": link}) + foreign_key_tables.append({**fk, **{"count": count, "link": link}}) return foreign_key_tables @@ -702,21 +708,21 @@ async def _row_flash_message(db, action, resolved, row=None): if label: label = _truncated_row_flash_label(label) if label and label != pk_label: - return f"{action} row {pk_label} ({label})" - return f"{action} row {pk_label}" + return "{} row {} ({})".format(action, pk_label, label) + return "{} row {}".format(action, pk_label) async def _resolve_row_and_check_permission(datasette, request, permission): - from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound + from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound try: resolved = await datasette.resolve_row(request) except DatabaseNotFound as e: - return False, Response.error([f"Database not found: {e.database_name}"], 404) + return False, _error(["Database not found: {}".format(e.database_name)], 404) except TableNotFound as e: - return False, Response.error([f"Table not found: {e.table}"], 404) + return False, _error(["Table not found: {}".format(e.table)], 404) except RowNotFound as e: - return False, Response.error([f"Record not found: {e.pk_values}"], 404) + return False, _error(["Record not found: {}".format(e.pk_values)], 404) # Ensure user has permission to delete this row if not await datasette.allowed( @@ -724,7 +730,7 @@ async def _resolve_row_and_check_permission(datasette, request, permission): resource=TableResource(database=resolved.db.name, table=resolved.table), actor=request.actor, ): - return False, Response.error(["Permission denied"], 403) + return False, _error(["Permission denied"], 403) return True, resolved @@ -748,9 +754,8 @@ class RowDeleteView(BaseView): try: await resolved.db.execute_write_fn(delete_row, request=request) - except Exception as e: # noqa: BLE001 - # TODO: narrow to expected write errors so Datasette bugs surface as 500s - return Response.error([str(e)], 400) + except Exception as e: + return _error([str(e)], 500) await self.ds.track_event( DeleteRowEvent( @@ -789,24 +794,18 @@ class RowUpdateView(BaseView): try: data = await request.json() except json.JSONDecodeError as e: - return Response.error([f"Invalid JSON: {e}"]) - except PayloadTooLarge as e: - return Response.error([str(e)], 413) + return _error(["Invalid JSON: {}".format(e)]) if not isinstance(data, dict): - return Response.error(["JSON must be a dictionary"]) + return _error(["JSON must be a dictionary"]) if "update" not in data or not isinstance(data["update"], dict): - return Response.error(["JSON must contain an update dictionary"]) + return _error(["JSON must contain an update dictionary"]) invalid_keys = set(data.keys()) - {"update", "return", "alter"} if invalid_keys: - return Response.error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) update = data["update"] - try: - update = decode_write_json_row(update) - except WriteJsonValueError as e: - return Response.error([str(e)], 400) # Validate column types from datasette.views.table import _validate_column_types @@ -815,7 +814,7 @@ class RowUpdateView(BaseView): self.ds, resolved.db.name, resolved.table, [update] ) if ct_errors: - return Response.error(ct_errors, 400) + return _error(ct_errors, 400) alter = data.get("alter") if alter and not await self.ds.allowed( @@ -823,7 +822,7 @@ class RowUpdateView(BaseView): resource=TableResource(database=resolved.db.name, table=resolved.table), actor=request.actor, ): - return Response.error(["Permission denied for alter-table"], 403) + return _error(["Permission denied for alter-table"], 403) def update_row(conn): sqlite_utils.Database(conn)[resolved.table].update( @@ -832,9 +831,8 @@ class RowUpdateView(BaseView): try: await resolved.db.execute_write_fn(update_row, request=request) - except Exception as e: # noqa: BLE001 - # TODO: narrow to expected write errors so Datasette bugs surface as 500s - return Response.error([str(e)], 400) + except Exception as e: + return _error([str(e)], 400) result = {"ok": True} returned_row = None @@ -843,7 +841,7 @@ class RowUpdateView(BaseView): resolved.sql, resolved.params, truncate=True ) returned_row = results.dicts()[0] - result["rows"] = [returned_row] + result["row"] = returned_row await self.ds.track_event( UpdateRowEvent( @@ -869,4 +867,4 @@ class RowUpdateView(BaseView): self.ds.INFO, ) - return Response.json(result, status=200, default=CustomJSONEncoder().default) + return Response.json(result, status=200) diff --git a/datasette/views/special.py b/datasette/views/special.py index a77f221f..3245bc13 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,25 +1,20 @@ import json import logging -import secrets -import urllib - -from datasette.events import CreateTokenEvent, LoginEvent, LogoutEvent from datasette.jump import JumpSQL, namespace_sql_params from datasette.plugins import pm +from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource +from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( - UNSTABLE_API_MESSAGE, actor_matches_allow, add_cors_headers, await_me_maybe, - error_body, - parse_size_limit, - tilde_decode, tilde_encode, + tilde_decode, ) -from datasette.utils.asgi import Forbidden, Response - from .base import BaseView, View +import secrets +import urllib logger = logging.getLogger(__name__) @@ -57,9 +52,9 @@ class JsonDataView(BaseView): if self.permission: await self.ds.ensure_permission(action=self.permission, actor=request.actor) if self.needs_request: - data = await await_me_maybe(self.data_callback(request)) + data = self.data_callback(request) else: - data = await await_me_maybe(self.data_callback()) + data = self.data_callback() # Return JSON or HTML depending on format parameter as_format = request.url_vars.get("format") @@ -67,8 +62,6 @@ class JsonDataView(BaseView): headers = {} if self.ds.cors: add_cors_headers(headers) - if isinstance(data, dict): - data = {"ok": True, **data} return Response.json(data, headers=headers) else: context = { @@ -181,7 +174,9 @@ class AutocompleteDebugView(BaseView): ) context.update( { - "autocomplete_url": f"{self.ds.urls.table(database_name, table_name)}/-/autocomplete", + "autocomplete_url": "{}/-/autocomplete".format( + self.ds.urls.table(database_name, table_name) + ), "label_column": await db.label_column_for_table(table_name), } ) @@ -297,12 +292,6 @@ class PermissionsDebugView(BaseView): response, status = await _check_permission_for_actor( self.ds, permission, parent, child, actor ) - if response.get("ok"): - response = { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, - **response, - } return Response.json(response, status=status) @@ -359,32 +348,29 @@ class AllowedResourcesView(BaseView): async def _allowed_payload(self, request, has_debug_permission): action = request.args.get("action") if not action: - return error_body("action parameter is required", 400), 400 + return {"error": "action parameter is required"}, 400 if action not in self.ds.actions: - return error_body(f"Unknown action: {action}", 404), 404 + return {"error": f"Unknown action: {action}"}, 404 actor = request.actor if isinstance(request.actor, dict) else None actor_id = actor.get("id") if actor else None parent_filter = request.args.get("parent") child_filter = request.args.get("child") if child_filter and not parent_filter: - return ( - error_body("parent must be provided when child is specified", 400), - 400, - ) + return {"error": "parent must be provided when child is specified"}, 400 try: - page = int(request.args.get("_page", "1")) - if page < 1: - raise ValueError + page = int(request.args.get("page", "1")) + page_size = int(request.args.get("page_size", "50")) except ValueError: - return error_body("_page must be a positive integer", 400), 400 - try: - page_size = parse_size_limit( - request.args.get("_size"), default=50, maximum=200 - ) - except ValueError as ex: - return error_body(str(ex), 400), 400 + return {"error": "page and page_size must be integers"}, 400 + if page < 1: + return {"error": "page must be >= 1"}, 400 + if page_size < 1: + return {"error": "page_size must be >= 1"}, 400 + max_page_size = 200 + if page_size > max_page_size: + page_size = max_page_size offset = (page - 1) * page_size # Use the simplified allowed_resources method @@ -420,14 +406,10 @@ class AllowedResourcesView(BaseView): row["reason"] = resource.reasons allowed_rows.append(row) - except Exception: # noqa: BLE001 - # Returns empty results if the catalog tables don't exist yet, but - # also swallows the AttributeError raised for instance-level actions - # such as view-instance, which have no resource_class. - # TODO: handle that case explicitly and narrow this to sqlite3.Error + except Exception: + # If catalog tables don't exist yet, return empty results return ( { - "ok": True, "action": action, "actor_id": actor_id, "page": page, @@ -452,17 +434,16 @@ class AllowedResourcesView(BaseView): def build_page_url(page_number): pairs = [] for key in request.args: - if key in {"_page", "_size"}: + if key in {"page", "page_size"}: continue for value in request.args.getlist(key): pairs.append((key, value)) - pairs.append(("_page", str(page_number))) - pairs.append(("_size", str(page_size))) + pairs.append(("page", str(page_number))) + pairs.append(("page_size", str(page_size))) query = urllib.parse.urlencode(pairs) return f"{request.path}?{query}" response = { - "ok": True, "action": action, "actor_id": actor_id, "page": page, @@ -504,29 +485,31 @@ class PermissionRulesView(BaseView): # JSON API - action parameter is required action = request.args.get("action") if not action: - return Response.error("action parameter is required", 400) + return Response.json({"error": "action parameter is required"}, status=400) if action not in self.ds.actions: - return Response.error(f"Unknown action: {action}", 404) + return Response.json({"error": f"Unknown action: {action}"}, status=404) actor = request.actor if isinstance(request.actor, dict) else None try: - page = int(request.args.get("_page", "1")) - if page < 1: - raise ValueError + page = int(request.args.get("page", "1")) + page_size = int(request.args.get("page_size", "50")) except ValueError: - return Response.error("_page must be a positive integer", 400) - try: - page_size = parse_size_limit( - request.args.get("_size"), default=50, maximum=200 + return Response.json( + {"error": "page and page_size must be integers"}, status=400 ) - except ValueError as ex: - return Response.error(str(ex), 400) + if page < 1: + return Response.json({"error": "page must be >= 1"}, status=400) + if page_size < 1: + return Response.json({"error": "page_size must be >= 1"}, status=400) + max_page_size = 200 + if page_size > max_page_size: + page_size = max_page_size offset = (page - 1) * page_size from datasette.utils.actions_sql import build_permission_rules_sql - union_sql, union_params, _restriction_sqls = await build_permission_rules_sql( + union_sql, union_params, restriction_sqls = await build_permission_rules_sql( self.ds, actor, action ) await self.ds.refresh_schemas() @@ -572,17 +555,16 @@ class PermissionRulesView(BaseView): def build_page_url(page_number): pairs = [] for key in request.args: - if key in {"_page", "_size"}: + if key in {"page", "page_size"}: continue for value in request.args.getlist(key): pairs.append((key, value)) - pairs.append(("_page", str(page_number))) - pairs.append(("_size", str(page_size))) + pairs.append(("page", str(page_number))) + pairs.append(("page_size", str(page_size))) query = urllib.parse.urlencode(pairs) return f"{request.path}?{query}" response = { - "ok": True, "action": action, "actor_id": (actor or {}).get("id") if actor else None, "page": page, @@ -603,17 +585,17 @@ class PermissionRulesView(BaseView): async def _check_permission_for_actor(ds, action, parent, child, actor): - """Shared logic for checking and explaining a permission decision.""" + """Shared logic for checking permissions. Returns a dict with check results.""" if action not in ds.actions: - return error_body(f"Unknown action: {action}", 404), 404 + return {"error": f"Unknown action: {action}"}, 404 if child and not parent: - return error_body("parent is required when child is provided", 400), 400 + return {"error": "parent is required when child is provided"}, 400 # Use the action's properties to create the appropriate resource object action_obj = ds.actions.get(action) if not action_obj: - return error_body(f"Unknown action: {action}", 400), 400 + return {"error": f"Unknown action: {action}"}, 400 # Global actions (no resource_class) don't have a resource if action_obj.resource_class is None: @@ -628,32 +610,18 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ - return error_body(f"Invalid action configuration: {action}", 500), 500 + return {"error": f"Invalid action configuration: {action}"}, 500 allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor) - from datasette.utils.actions_sql import explain_permission_for_resource - - explanation = await explain_permission_for_resource( - datasette=ds, - actor=actor, - action=action, - parent=parent, - child=child, - ) - response = { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, "action": action, "allowed": bool(allowed), - "actor": actor, "resource": { "parent": parent, "child": child, "path": _resource_path(parent, child), }, - "explanation": explanation, } if actor and "id" in actor: @@ -671,25 +639,11 @@ class PermissionCheckView(BaseView): as_format = request.url_vars.get("format") if not as_format: - actions = [ - { - "name": action.name, - "description": action.description, - "takes_parent": action.takes_parent, - "takes_child": action.takes_child, - "also_requires": action.also_requires, - } - for action in sorted( - self.ds.actions.values(), key=lambda action: action.name - ) - ] return await self.render( ["debug_check.html"], request, { - "actions": actions, - "actor_json": request.args.get("actor") - or json.dumps(request.actor, indent=2), + "sorted_actions": sorted(self.ds.actions.keys()), "has_debug_permission": True, }, ) @@ -697,22 +651,13 @@ class PermissionCheckView(BaseView): # JSON API - action parameter is required action = request.args.get("action") if not action: - return Response.error("action parameter is required", 400) + return Response.json({"error": "action parameter is required"}, status=400) parent = request.args.get("parent") child = request.args.get("child") - actor = request.actor - actor_json = request.args.get("actor") - if actor_json is not None: - try: - actor = json.loads(actor_json) - except json.JSONDecodeError as ex: - return Response.error(f"Invalid actor JSON: {ex}", 400) - if actor is not None and not isinstance(actor, dict): - return Response.error("actor must be a JSON object or null", 400) response, status = await _check_permission_for_actor( - self.ds, action, parent, child, actor + self.ds, action, parent, child, request.actor ) return Response.json(response, status=status) @@ -939,7 +884,7 @@ class ApiExplorerView(BaseView): tables.append({"name": table, "links": table_links}) table_links.append( { - "label": f"Get rows for {table}", + "label": "Get rows for {}".format(table), "method": "GET", "path": self.ds.urls.table(name, table, format="json"), } @@ -959,7 +904,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/insert", "method": "POST", - "label": f"Insert rows into {table}", + "label": "Insert rows into {}".format(table), "json": { "rows": [ { @@ -973,7 +918,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/upsert", "method": "POST", - "label": f"Upsert rows into {table}", + "label": "Upsert rows into {}".format(table), "json": { "rows": [ { @@ -1003,7 +948,7 @@ class ApiExplorerView(BaseView): table_links.append( { "path": self.ds.urls.table(name, table) + "/-/drop", - "label": f"Drop table {table}", + "label": "Drop table {}".format(table), "json": {"confirm": False}, "method": "POST", } @@ -1020,7 +965,7 @@ class ApiExplorerView(BaseView): database_links.append( { "path": self.ds.urls.database(name) + "/-/create", - "label": f"Create table in {name}", + "label": "Create table in {}".format(name), "json": { "table": "new_table", "columns": [ @@ -1253,7 +1198,7 @@ class JumpView(BaseView): match["display_name"] = row["display_name"] matches.append(match) - return Response.json({"ok": True, "matches": matches, "truncated": truncated}) + return Response.json({"matches": matches, "truncated": truncated}) class SchemaBaseView(BaseView): @@ -1275,7 +1220,7 @@ class SchemaBaseView(BaseView): headers = {} if self.ds.cors: add_cors_headers(headers) - return Response.json({"ok": True, **data}, headers=headers) + return Response.json(data, headers=headers) def format_error_response(self, error_message, format_, status=404): """Format error response based on requested format.""" @@ -1284,7 +1229,7 @@ class SchemaBaseView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response.json( - error_body(error_message, status), status=status, headers=headers + {"ok": False, "error": error_message}, status=status, headers=headers ) else: return Response.text(error_message, status=status) @@ -1360,17 +1305,17 @@ class DatabaseSchemaView(SchemaBaseView): database_name = request.url_vars["database"] format_ = request.url_vars.get("format") or "html" - # Permission check comes first, so actors without view-database - # cannot distinguish existing databases from missing ones + # Check if database exists + if database_name not in self.ds.databases: + return self.format_error_response("Database not found", format_) + + # Check view-database permission await self.ds.ensure_permission( action="view-database", resource=DatabaseResource(database=database_name), actor=request.actor, ) - if database_name not in self.ds.databases: - return self.format_error_response("Database not found", format_) - schema = await self.get_database_schema(database_name) if format_ == "json": @@ -1404,9 +1349,6 @@ class TableSchemaView(SchemaBaseView): actor=request.actor, ) - if database_name not in self.ds.databases: - return self.format_error_response("Database not found", format_) - # Get schema for the table db = self.ds.databases[database_name] result = await db.execute( diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 0bbe9f38..2753f876 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -2,10 +2,10 @@ from urllib.parse import parse_qsl, urlencode from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict -from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3, tilde_decode +from datasette.utils import sqlite3, tilde_decode from datasette.utils.asgi import Response -from .base import BaseView +from .base import BaseView, _error from .query_helpers import ( QueryValidationError, _as_bool, @@ -34,14 +34,12 @@ class QueryParametersView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing( - Response.error(["Permission denied: need execute-sql"], 403) - ) + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - Response.error( + _error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) @@ -49,16 +47,8 @@ class QueryParametersView(BaseView): try: parameters = _derived_query_parameters(request.args.get("sql") or "") except QueryValidationError as ex: - return _block_framing(Response.error([ex.message], ex.status)) - return _block_framing( - Response.json( - { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, - "parameters": parameters, - } - ) - ) + return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.json({"ok": True, "parameters": parameters})) def _query_list_url(path, query_string, *, set_args=None, remove_args=None): @@ -92,12 +82,11 @@ class QueryListView(BaseView): limit = _query_list_limit( request.args.get("_size"), default=20 if format_ == "html" else 50, - maximum=self.ds.max_returned_rows, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") is_private = _as_optional_bool(request.args.get("is_private"), "is_private") except QueryValidationError as ex: - return Response.error([ex.message], ex.status) + return _error([ex.message], ex.status) page = await self.ds.list_queries( database, @@ -122,9 +111,9 @@ class QueryListView(BaseView): if key != "_next" ] pairs.append(("_next", page.next)) - next_url = self.ds.absolute_url( - request, - f"{request.path}?{urlencode(pairs)}", + next_url = "{}?{}".format( + query_list_path, + urlencode(pairs), ) current_filters = { @@ -210,6 +199,7 @@ class QueryListView(BaseView): "queries": page.queries, "next": page.next, "next_url": next_url, + "has_more": page.has_more, "limit": page.limit, "show_private_note": any(query.is_private for query in page.queries), "show_trusted_note": any(query.is_trusted for query in page.queries), @@ -308,30 +298,28 @@ class QueryCreateAnalyzeView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing( - Response.error(["Permission denied: need execute-sql"], 403) - ) + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) if not await self.ds.allowed( action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing( - Response.error(["Permission denied: need store-query"], 403) - ) + return _block_framing(_error(["Permission denied: need store-query"], 403)) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - Response.error( + _error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) ) sql = request.args.get("sql") or "" - analysis = await _query_create_analysis_data(self.ds, db, sql, request.actor) - analysis["unstable"] = UNSTABLE_API_MESSAGE - return _block_framing(Response.json(analysis)) + return _block_framing( + Response.json( + await _query_create_analysis_data(self.ds, db, sql, request.actor) + ) + ) class QueryStoreView(QueryCreateView): @@ -358,13 +346,13 @@ class QueryStoreView(QueryCreateView): resource=DatabaseResource(db.name), actor=request.actor, ): - return Response.error(["Permission denied: need execute-sql"], 403) + return _error(["Permission denied: need execute-sql"], 403) if not await self.ds.allowed( action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return Response.error(["Permission denied: need store-query"], 403) + return _error(["Permission denied: need store-query"], 403) is_json = False query_data = {} @@ -381,7 +369,7 @@ class QueryStoreView(QueryCreateView): return await self._error_response( request, db, query_data, ex.message, ex.status ) - return Response.error([ex.message], ex.status) + return _error([ex.message], ex.status) prepared.pop("analysis") name = prepared.pop("name") @@ -390,18 +378,13 @@ class QueryStoreView(QueryCreateView): except sqlite3.IntegrityError as ex: if not is_json and isinstance(query_data, dict): return await self._error_response(request, db, query_data, str(ex), 400) - return Response.error([str(ex)], 400) + return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) assert query is not None if is_json: return Response.json( - { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, - "query": stored_query_to_dict(query), - }, - status=201, + {"ok": True, "query": stored_query_to_dict(query)}, status=201 ) self.ds.add_message(request, "Query saved", self.ds.INFO) return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) @@ -415,20 +398,14 @@ class QueryDefinitionView(BaseView): query_name = tilde_decode(request.url_vars["query"]) query = await self.ds.get_query(db.name, query_name) if query is None: - return Response.error([f"Query not found: {query_name}"], 404) + return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="view-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return Response.error(["Permission denied"], 403) - return Response.json( - { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, - "query": stored_query_to_dict(query), - } - ) + return _error(["Permission denied"], 403) + return Response.json({"ok": True, "query": stored_query_to_dict(query)}) class QueryUpdateView(BaseView): @@ -439,17 +416,15 @@ class QueryUpdateView(BaseView): query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return Response.error(["Permission denied: need update-query"], 403) + return _error(["Permission denied: need update-query"], 403) if existing.is_trusted: - return Response.error( - ["Trusted queries cannot be updated using the API"], 403 - ) + return _error(["Trusted queries cannot be updated using the API"], 403) try: data, _ = await _json_or_form_payload(request) @@ -475,7 +450,7 @@ class QueryUpdateView(BaseView): self.ds, request, db, existing, update ) except QueryValidationError as ex: - return Response.error([ex.message], ex.status) + return _error([ex.message], ex.status) await self.ds.update_query(db.name, query_name, **update_kwargs) if data.get("return"): @@ -532,32 +507,32 @@ class QueryEditView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return _error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ) if existing.is_trusted: - return Response.error(["Trusted queries cannot be edited"], 403) + return _error(["Trusted queries cannot be edited"], 403) return await self._render_form(request, db, existing) async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return Response.error(["Permission denied: need update-query"], 403) + return _error(["Permission denied: need update-query"], 403) if existing.is_trusted: - return Response.error(["Trusted queries cannot be edited"], 403) + return _error(["Trusted queries cannot be edited"], 403) data, _ = await _json_or_form_payload(request) if not isinstance(data, dict): - return Response.error(["Invalid form submission"], 400) + return _error(["Invalid form submission"], 400) sql = data.get("sql") sql = existing.sql if sql is None else sql.strip() title = data.get("title") or "" @@ -629,16 +604,12 @@ class QueryDeleteView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return _error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="delete-query", resource=QueryResource(db.name, query_name), actor=request.actor, ) - if existing.is_trusted: - return Response.error( - ["Trusted queries cannot be deleted using the API"], 403 - ) return await self.render( ["query_delete.html"], request, @@ -653,25 +624,21 @@ class QueryDeleteView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="delete-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return Response.error(["Permission denied: need delete-query"], 403) - if existing.is_trusted: - return Response.error( - ["Trusted queries cannot be deleted using the API"], 403 - ) + return _error(["Permission denied: need delete-query"], 403) - _data, is_json = await _json_or_form_payload(request) + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) if is_json: return Response.json({"ok": True}) self.ds.add_message( request, - f"Query “{existing.title or query_name}” deleted", + "Query “{}” deleted".format(existing.title or query_name), self.ds.INFO, ) return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/datasette/views/table.py b/datasette/views/table.py index 7c814b27..1fc151e6 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,63 +3,54 @@ import itertools import json import urllib import urllib.parse -from dataclasses import dataclass, field import markupsafe -import sqlite_utils -from datasette import tracer from datasette.column_types import SQLiteType -from datasette.database import QueryInterrupted +from datasette.extras import extra_names_from_request +from datasette.plugins import pm from datasette.events import ( AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, ) -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.filters import Filters -from datasette.plugins import pm +from datasette.database import QueryInterrupted +from datasette import tracer from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( - CustomJSONEncoder, - CustomRow, - InvalidSql, - WriteJsonValueError, add_cors_headers, - append_querystring, await_me_maybe, call_with_supported_arguments, + CustomRow, + append_querystring, compound_keys_after_sql, - decode_write_json_rows, + format_bytes, + make_slot_function, + tilde_encode, escape_sqlite, filters_should_redirect, - format_bytes, is_url, - make_slot_function, path_from_row_pks, path_with_added_args, path_with_format, path_with_removed_args, path_with_replaced_args, - sqlite3, - tilde_encode, to_css_class, truncate_url, urlsafe_components, value_as_boolean, + InvalidSql, + sqlite3, ) -from datasette.utils.asgi import ( - BadRequest, - Forbidden, - NotFound, - PayloadTooLarge, - Request, - Response, -) +from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response +from datasette.filters import Filters +import sqlite_utils +from dataclasses import dataclass, field +from datasette.extras import ExtraScope from . import Context, from_extra -from .base import BaseView, DatasetteError, stream_csv +from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_create_alter import ( ALTER_TABLE_COLUMN_TYPES, @@ -71,7 +62,6 @@ from .table_create_alter import ( from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, - count_is_truncated, precompute_database_action_permissions, precompute_table_action_permissions, resolve_table_extras, @@ -106,6 +96,7 @@ class TableContext(Context): human_description_en: str = from_extra() is_view: bool = from_extra() metadata: dict = from_extra() + next_url: str = from_extra() primary_keys: list = from_extra() private: bool = from_extra() query: dict = from_extra() @@ -122,11 +113,6 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) - next_url: str = field( - metadata={ - "help": "Full URL for the next page of results, or None if there are no more pages. See :ref:`json_api_pagination`." - } - ) count_truncated: bool = field( metadata={ "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." @@ -219,7 +205,7 @@ class TableContext(Context): ) table_insert_ui: dict = field( metadata={ - "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys." + "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys." } ) table_alter_ui: dict = field( @@ -494,15 +480,8 @@ async def _table_insert_ui( ): return None - can_update = await datasette.allowed( - action="update-row", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ) - column_types_map = await datasette.get_column_types(database_name, table_name) columns = [] - bulk_columns = [] column_details = await db.table_column_details(table_name) for column in column_details: if column.hidden: @@ -513,40 +492,32 @@ async def _table_insert_ui( and len(pks) == 1 and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER ) - column_type = column_types_map.get(column.name) - column_data = { - "name": column.name, - "sqlite_type": _column_sqlite_type_for_insert_form(column), - "notnull": column.notnull, - "default": column.default_value, - "has_default": column.default_value is not None, - "is_pk": is_pk, - "is_auto_pk": is_auto_pk, - "value_kind": _column_value_kind_for_insert_form(column), - "column_type": ( - {"type": column_type.name, "config": column_type.config} - if column_type is not None - else None - ), - } - bulk_columns.append(column_data) if is_auto_pk: continue - columns.append(column_data) + column_type = column_types_map.get(column.name) + columns.append( + { + "name": column.name, + "sqlite_type": _column_sqlite_type_for_insert_form(column), + "notnull": column.notnull, + "default": column.default_value, + "has_default": column.default_value is not None, + "is_pk": is_pk, + "value_kind": _column_value_kind_for_insert_form(column), + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + ) - data = { - "path": f"{datasette.urls.table(database_name, table_name)}/-/insert", + return { + "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, - "bulkColumns": bulk_columns, "primaryKeys": pks, - "maxInsertRows": datasette.setting("max_insert_rows"), } - if can_update: - data["upsertPath"] = ( - f"{datasette.urls.table(database_name, table_name)}/-/upsert" - ) - return data async def _table_alter_ui( @@ -603,7 +574,7 @@ async def _table_alter_ui( columns.append(column_data) data = { - "path": f"{datasette.urls.table(database_name, table_name)}/-/alter", + "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, "primaryKeys": pks, @@ -629,7 +600,9 @@ async def _table_alter_ui( actor=request.actor, ) if can_drop_table: - data["dropPath"] = f"{datasette.urls.table(database_name, table_name)}/-/drop" + data["dropPath"] = "{}/-/drop".format( + datasette.urls.table(database_name, table_name) + ) return data @@ -725,10 +698,12 @@ async def display_columns_and_rows( row_label = row_label_from_label_column(row, label_column) row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = f"{pk_path} {row_label}" + row_action_label = "{} {}".format(pk_path, row_label) table_path = datasette.urls.table(database_name, table_name) - row_link = ( - f'{markupsafe.escape(pk_path)!s}' + row_link = '{flat_pks}'.format( + table_path=table_path, + flat_pks=str(markupsafe.escape(pk_path)), + flat_pks_quoted=row_path, ) edit_icon = ( '/-/upsert`` API when the actor has both :ref:`insert-row ` and :ref:`update-row ` permissions. (:pr:`2813`) -- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`) -- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text. (:issue:`2806`, :pr:`2822`) -- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. (:issue:`2806`, :pr:`2822`) -- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. -- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits. (:issue:`2823`) -- Row pages for tables with compound primary keys now return a ``400`` error instead of a ``500`` error when the URL row identifier does not contain the correct number of primary key values. Thanks, `Zain Dana Harper `__. (:issue:`2811`, :pr:`2815`) -- The :ref:`execute-write-sql ` interface now supports ``CREATE VIEW`` and ``DROP VIEW`` statements, gated by the new :ref:`create-view ` and :ref:`drop-view ` permissions. (:issue:`2819`, :pr:`2818`) -- Saved-query SQL analysis now handles recursive CTEs, fixing a bug where storing a valid read-only recursive query could be disabled by SQLite's internal ``SQLITE_RECURSIVE`` authorizer callback. (:issue:`2809`, :pr:`2812`) -- ``named_parameters()`` now correctly ignores SQLite comment markers that appear inside string literals, so query forms no longer drop later ``:named`` parameters from SQL such as ``select '--' || :name``. Thanks, `JSap0914 `__. (:pr:`2783`) -- Datasette's internal database schema is now managed using `sqlite-utils migrations `__, using the new dependency on ``sqlite-utils>=4.0``. (:issue:`2827`) -- ``datasette.utils.CustomJSONEncoder`` is now documented as a public API for plugins that need to serialize Datasette values to JSON. Thanks, `Chris Amico `__. (:issue:`1983`, :pr:`1996`) - -This release also includes the results of a `detailed consistency review `__ of Datasette's JSON API in preparation for the 1.0 stable release. Several of these changes are backwards-incompatible with previous 1.0 alphas. The new :ref:`API stability documentation ` describes exactly which parts of the JSON API are covered by the 1.0 stability promise. - -JSON API: breaking changes -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- JSON error responses now use a single canonical format across every endpoint: ``{"ok": false, "error": "...", "errors": [...], "status": 400}``. The ``error`` key joins all error messages together, ``errors`` is the full list of messages and ``status`` always matches the HTTP status code. The legacy ``title`` key is no longer included in JSON errors (it remains available to the HTML error template), and endpoints that previously returned bare ``{"error": ...}`` objects have been updated. See :ref:`json_api_errors`. -- Every JSON object success response now includes ``"ok": true``, including introspection endpoints such as ``/-/versions`` and ``/-/settings``. -- ``/-/plugins.json``, ``/-/databases.json`` and ``/-/actions.json`` now return objects - ``{"ok": true, "plugins": [...]}`` and equivalents - instead of top-level JSON arrays, so these responses can gain additional keys in the future without a breaking change. The ``datasette plugins`` CLI command still outputs a plain array. -- ``/-/databases`` now only lists databases the current actor is allowed to view. It previously listed every attached database, including their filesystem paths, to any actor with ``view-instance``. -- Requests with an invalid or expired ``Authorization: Bearer`` token now receive a ``401`` status with the standard error body and a ``WWW-Authenticate: Bearer error="invalid_token"`` header, instead of being silently treated as unauthenticated. Bearer tokens that no registered token handler recognizes are still ignored, so authentication plugins with their own token formats keep working. Plugin :ref:`token handlers ` can raise the new ``datasette.TokenInvalid`` exception to trigger the same behavior. -- Permission errors for JSON requests now return the standard JSON error format with a ``403`` status. The default forbidden handling previously rendered an HTML error page even for ``.json`` requests. -- ``POST`` to a write canned query now returns a ``400`` error when the SQL fails to execute, instead of a ``200`` status with ``"ok": false`` in the body. The error response includes the standard error keys plus a ``"redirect"`` key. -- The :ref:`row update API ` with ``"return": true`` now responds with a ``"rows"`` list, matching insert and upsert, instead of a singular ``"row"`` object. -- Row delete write failures - such as a constraint violation raised by a trigger - now return ``400`` instead of ``500``, matching the other write endpoints. -- ``//-/query.json`` with a missing or blank ``?sql=`` parameter now returns a ``400`` error, as the CSV format already did, instead of a ``200`` with empty rows. -- Unknown ``?_extra=`` names now return a ``400`` error for JSON and other data formats, instead of being silently ignored. HTML pages continue to ignore unknown names. -- Table JSON responses now include ``next_url`` alongside ``next`` by default - both are ``null`` on the final page. The now-redundant ``?_extra=next_url`` parameter has been removed. -- The stored query list JSON no longer includes ``has_more`` - ``"next": null`` is the end-of-results signal across the whole API. This change also uncovered and fixed a bug where the query list ``next_url`` pointed at the HTML page and was a relative path; it is now an absolute URL that preserves the requested format. -- Stored query JSON objects no longer duplicate the list of parameter names as both ``params`` and ``parameters`` - only ``parameters`` remains. The query create and update APIs no longer accept ``params`` as an input alias either; ``params`` is still the documented key for :ref:`queries defined in configuration `. -- Page size parameters are now consistent across the API: the stored query lists accept ``?_size=max`` and return a ``400`` error for values over the maximum instead of silently clamping them, and the ``/-/allowed`` and ``/-/rules`` permission debug endpoints renamed their ``page`` and ``page_size`` parameters to ``_page`` and ``_size``, matching the underscore grammar used by every other Datasette system parameter. -- ``/-/threads`` now requires the ``permissions-debug`` permission, since it exposes runtime internals such as file paths. It previously only required ``view-instance``. -- Trusted stored queries - those defined in configuration - can no longer be deleted through the JSON API or web interface, matching the existing restriction on editing them. -- The ``//-/schema`` endpoints now check the ``view-database`` permission before checking whether the database exists, so unauthorized actors can no longer probe for the existence of databases. -- SQL time limit errors in JSON responses are now a plain text message. The error string previously embedded an HTML fragment. -- The undocumented homepage JSON at ``/.json`` now returns ``databases`` as a list of objects rather than an object keyed by database name, matching every other collection in the API. -- The legacy ``.jsono`` format extension, long since superseded by ``?_shape=``, has been removed. - -JSON API: other improvements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- The :ref:`write API ` endpoints now parse the request body as JSON regardless of the ``Content-Type`` header, so ``curl -d`` invocations work without remembering to set it. Invalid JSON is a ``400`` error. Cross-site request forgery remains prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` checks. This also fixes a ``500`` error from the insert API when the ``Content-Type`` header was missing entirely. -- New ``Response.error(messages, status=400)`` helper for plugins that need to return a JSON error in Datasette's standard format. See :ref:`internals_response`. -- New ``count_truncated`` extra for table JSON, included automatically whenever ``count`` is requested. ``true`` means the count reached Datasette's counting limit and the real number of rows may be higher. See :ref:`json_api_extra`. -- JSON endpoints that are not part of the documented stable API now declare themselves with an ``"unstable"`` key in their responses. -- New documentation covering the grammar for :ref:`boolean query string arguments `, the reason :ref:`upsert ` returns ``200`` where insert returns ``201``, and advice for plugin authors on :ref:`naming secret configuration keys ` so that ``/-/config`` redacts them automatically. - .. _v1_0_a35: 1.0a35 (2026-06-23) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2302f742..7ca88c4e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -244,9 +244,6 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam custom query (default=1000) max_insert_rows Maximum rows that can be inserted at a time using the bulk insert API (default=100) - max_post_body_bytes Maximum size in bytes for a POST body read into - memory, e.g. JSON API requests - set 0 to disable - this limit (default=2097152) num_sql_threads Number of threads in the thread pool for executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds diff --git a/docs/conf.py b/docs/conf.py index 2a5c1439..5dd06b57 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # # Datasette documentation build configuration file, created by # sphinx-quickstart on Thu Nov 16 06:50:13 2017. diff --git a/docs/installation.rst b/docs/installation.rst index ceec7f23..33d3d6a1 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -17,6 +17,13 @@ If you want to start making contributions to the Datasette project by installing Basic installation ================== +.. _installation_datasette_desktop: + +Datasette Desktop for Mac +------------------------- + +`Datasette Desktop `__ is a packaged Mac application which bundles Datasette together with Python and allows you to install and run Datasette directly on your laptop. This is the best option for local installation if you are not comfortable using the command line. + .. _installation_homebrew: Using Homebrew diff --git a/docs/internals.rst b/docs/internals.rst index d2bd46ef..c826de1a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -52,9 +52,6 @@ The request object is passed to various plugin hooks. It represents an incoming ``.actor`` - dictionary (str -> Any) or None The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated. -``.max_post_body_bytes`` - integer - The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument. - The object also has the following awaitable methods: ``await request.form(files=False, ...)`` - FormData @@ -112,11 +109,9 @@ The object also has the following awaitable methods: ``await request.json()`` - Any Returns the parsed JSON body of a request submitted by ``POST``. -``await request.post_body(max_bytes=None)`` - bytes +``await request.post_body()`` - bytes Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data. - The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them. - And a class method that can be used to create fake request objects for use in tests: ``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)`` @@ -284,7 +279,7 @@ For example: content_type="application/xml; charset=utf-8", ) -The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)``, ``Response.error(...)`` or ``Response.redirect(...)`` helper methods: +The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods: .. code-block:: python @@ -295,8 +290,6 @@ The quickest way to create responses is using the ``Response.text(...)``, ``Resp text_response = Response.text( "This will become utf-8 encoded text" ) - # A JSON error in Datasette's standard error format: - error_response = Response.error("Cannot do that", 400) # Redirects are served as 302, unless you pass status=301: redirect_response = Response.redirect( "https://latest.datasette.io/" @@ -306,8 +299,6 @@ Each of these responses will use the correct corresponding content-type - ``text Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above. -``Response.error(messages, status=400)`` returns a JSON error in the :ref:`standard Datasette error format `. ``messages`` can be a single string or a list of strings. Use this for JSON-only endpoints; if your error should content-negotiate between JSON and HTML, raise ``Forbidden``, ``NotFound``, ``BadRequest`` or ``DatasetteError`` instead and Datasette's error handling will build the appropriate response. - .. _internals_response_asgi_send: Returning a response with .asgi_send(send) @@ -2023,8 +2014,8 @@ Example usage: .. _database_execute_write: -await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True) --------------------------------------------------------------------------------------------------------------------------- +await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10) +-------------------------------------------------------------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -2059,9 +2050,7 @@ If you need to retrieve every row returned by a statement, pass ``return_all=Tru If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. -Each call to ``execute_write()`` will be executed inside a transaction. Pass -``transaction=False`` for statements such as ``VACUUM`` that cannot run inside -a transaction. +Each call to ``execute_write()`` will be executed inside a transaction. .. _database_execute_write_script: @@ -2365,14 +2354,6 @@ The internal database schema is as follows: .. code-block:: sql - CREATE TABLE "_sqlite_migrations" ( - "id" INTEGER PRIMARY KEY, - "migration_set" TEXT, - "name" TEXT, - "applied_at" TEXT - ); - CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name" - ON "_sqlite_migrations" ("migration_set", "name"); CREATE TABLE catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, @@ -2598,13 +2579,6 @@ Async version of :ref:`call_with_supported_arguments `. - -The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise `, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases. - .. _JsonDataView_metadata: /-/metadata @@ -41,7 +37,6 @@ Shows the version of Datasette, Python and SQLite. `Versions example ` for this instance of Datasette. T .. code-block:: json { - "ok": true, "settings": { "template_debug": true, "trace_debug": true, @@ -136,47 +129,20 @@ Any keys that include the one of the following substrings in their names will be /-/databases ------------ -Shows currently attached databases that the current actor is allowed to view, based on the ``view-database`` permission. `Databases example `_: +Shows currently attached databases. `Databases example `_: .. code-block:: json - { - "ok": true, - "databases": [ - { - "hash": null, - "is_memory": false, - "is_mutable": true, - "name": "fixtures", - "path": "fixtures.db", - "size": 225280 - } - ] - } - -.. _JsonDataView_actions: - -/-/actions ----------- - -Shows all actions registered with the permission system, including those added by plugins. Requires the ``permissions-debug`` permission. - -.. code-block:: json - - { - "ok": true, - "actions": [ - { - "name": "view-instance", - "abbr": "vi", - "description": "View Datasette instance", - "takes_parent": false, - "takes_child": false, - "resource_class": null, - "also_requires": null - } - ] - } + [ + { + "hash": null, + "is_memory": false, + "is_mutable": true, + "name": "fixtures", + "path": "fixtures.db", + "size": 225280 + } + ] .. _JumpView: @@ -194,7 +160,6 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name. .. code-block:: json { - "ok": true, "matches": [ { "name": "fixtures", @@ -223,7 +188,6 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``: .. code-block:: json { - "ok": true, "matches": [ { "name": "fixtures: facetable", @@ -251,12 +215,11 @@ Without those query string arguments, the page lists up to five tables with dete /-/threads ---------- -Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``permissions-debug`` permission, since it exposes runtime internals. `Threads example `_: +Shows details of threads and ``asyncio`` tasks. `Threads example `_: .. code-block:: json { - "ok": true, "num_threads": 2, "threads": [ { @@ -288,7 +251,6 @@ Shows the currently authenticated actor. Useful for debugging Datasette authenti .. code-block:: json { - "ok": true, "actor": { "id": 1, "username": "some-user" diff --git a/docs/json_api.rst b/docs/json_api.rst index a96fd73d..eca22fdc 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -9,53 +9,6 @@ through the Datasette user interface can also be accessed as JSON via the API. To access the API for a page, either click on the ``.json`` link on that page or edit the URL and add a ``.json`` extension to it. -.. _json_api_stability: - -API stability -------------- - -Datasette 1.0 makes a stability promise for its JSON API: the endpoints, -parameters and response keys documented here and on the pages this -documentation links to will not change in backwards-incompatible ways for -the duration of the 1.x release series. - -Stability means: - -- Documented endpoints will keep their URLs, methods, parameters and - permission requirements. -- Documented response keys will keep their names and types. New keys may be - **added** in any release - clients should ignore keys they do not - recognize. -- The documented ``?_extra=`` names, ``?_shape=`` values and - :ref:`column filter operators ` are stable. -- Pagination tokens - the ``"next"`` key and ``?_next=`` parameter - are - **opaque strings**. Pass them back exactly as you received them; their - internal structure is not part of the API and can change at any time. -- The :ref:`standard error format ` and the - :ref:`API token format and restriction semantics ` are - stable, including the action abbreviations stored inside signed tokens. - -Some JSON endpoints are **exempt** from this promise: - -- Endpoints that are not documented include this marker key in their - responses and can change at any time:: - - "unstable": "This API is not part of Datasette's stable interface and may change at any time" - - This currently covers the instance homepage (``/.json``), the stored - query ``analyze``/``store``/``definition`` endpoints, ``/-/query/parameters``, - ``/-/execute-write/analyze`` and the JSON returned by the ``/-/permissions`` - debug playground. -- Debug and support endpoints are documented so you can use them, but their - JSON shapes are not frozen: :ref:`/-/threads `, - :ref:`/-/actions `, - the :ref:`permission debug endpoints ` - (``/-/allowed``, ``/-/rules``, ``/-/check``) and the - :ref:`table autocomplete endpoint `. -- Response keys explicitly labeled as unstable in this documentation, such - as the ``"analysis"`` block returned by :ref:`execute-write ` - and the ``debug`` and ``request`` extras. - .. _json_api_default: Default representation @@ -89,49 +42,13 @@ looks like this: "truncated": false } -``"ok"`` is always ``true`` if an error did not occur. Every Datasette JSON endpoint that returns an object includes this key on success. +``"ok"`` is always ``true`` if an error did not occur. The ``"rows"`` key is a list of objects, each one representing a row. The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). -For table pages, two additional keys are present: ``"next"``, an opaque token that can be used to retrieve the next page using ``?_next=TOKEN``, and ``"next_url"``, the full URL of that next page. Both are ``null`` on the final page. See :ref:`json_api_pagination`. - -.. _json_api_errors: - -Error responses ---------------- - -Every JSON error response from Datasette uses the same format: - -.. code-block:: json - - { - "ok": false, - "error": "Table not found", - "errors": [ - "Table not found" - ], - "status": 404 - } - -- ``"ok"`` is always ``false`` for an error. -- ``"errors"`` is a list of one or more error message strings. Endpoints that - validate multiple things at once - such as the :ref:`insert API ` - - may return several messages here. -- ``"error"`` is all of those messages joined with ``"; "``, for - convenience when displaying a single string. -- ``"status"`` matches the HTTP status code of the response. - -Some endpoints add extra context keys. For example, a SQL error from a -:ref:`custom query ` also includes the empty -``"rows"`` and ``"truncated"`` keys of the response it was unable to -produce. - -Permission errors use the same format: a request that fails a permission -check receives a ``403`` with this JSON error body when the URL ends in -``.json`` or the request sends an ``Accept: application/json`` or -``Content-Type: application/json`` header. +For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. .. _json_api_custom_sql: @@ -175,7 +92,6 @@ options: { "ok": true, "next": null, - "next_url": null, "rows": [ [3, "Detroit"], [2, "Los Angeles"], @@ -276,10 +192,6 @@ Here is an example Python function built using `requests `, for example ``{"ok": false, "error": "Unknown _extra: nope", ...}``. + ?_extra=columns&_extra=count,next_url .. [[[cog from json_api_doc import table_extras @@ -356,15 +266,6 @@ The available table extras are listed below. 15 -``count_truncated`` - True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count. (May execute additional queries.) - - ``GET /fixtures/facetable.json?_extra=count,count_truncated`` - - .. code-block:: json - - false - ``count_sql`` SQL query string used to calculate the total count for the current table view, including active filters. @@ -437,6 +338,17 @@ The available table extras are listed below. "where state = \"CA\" sorted by pk" +``next_url`` + Full URL for the next page of results + + ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + + ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. + + .. code-block:: json + + "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" + ``columns`` List of column names returned by this table, row or query. @@ -490,25 +402,6 @@ The available table extras are listed below. "pk" ] -``column_details`` - SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) - - ``GET /fixtures/binary_data.json?_size=0&_extra=column_details`` - - .. code-block:: json - - { - "data": { - "type": "BLOB", - "sqlite_type": "BLOB", - "notnull": false, - "default": null, - "is_pk": false, - "pk_position": 0, - "hidden": 0 - } - } - ``display_columns`` Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys. @@ -914,25 +807,6 @@ The following extras are available for row JSON responses. "id" ] -``column_details`` - SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) - - ``GET /fixtures/binary_data/1.json?_extra=column_details`` - - .. code-block:: json - - { - "data": { - "type": "BLOB", - "sqlite_type": "BLOB", - "notnull": false, - "default": null, - "is_pk": false, - "pk_position": 0, - "hidden": 0 - } - } - ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) @@ -1263,6 +1137,7 @@ The following extras are available for arbitrary SQL query responses and stored, "description_html": null, "hide_sql": false, "fragment": null, + "params": [], "parameters": [], "is_write": false, "is_private": false, @@ -1657,10 +1532,6 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. -The request body is always parsed as JSON, regardless of the request's ``Content-Type`` header - a body that is not valid JSON returns a ``400`` error. Cross-site request forgery is prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` header checks rather than by content type requirements. - -The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data. - .. _ExecuteWriteView: Executing write SQL @@ -1694,7 +1565,7 @@ Unsupported SQL operations are rejected by default. ``VACUUM`` is not allowed in A successful response includes a message, the SQLite ``rowcount``, a ``"rows"`` list, a ``"truncated"`` flag and a summary of the operations that were executed: -The shape of the ``"analysis"`` block is not part of the :ref:`stable API ` and may change in future Datasette releases. +The shape of the ``"analysis"`` block is not yet considered a stable API and may change in future Datasette releases. .. code-block:: json @@ -1754,17 +1625,15 @@ the execute-write returning row limit, which defaults to 10: ] } -Errors use the :ref:`standard Datasette error format `: +Errors use the standard Datasette error format: .. code-block:: json { "ok": false, - "error": "Permission denied: need execute-write-sql", "errors": [ "Permission denied: need execute-write-sql" - ], - "status": 403 + ] } .. _TableInsertView: @@ -1791,8 +1660,6 @@ A single row can be inserted using the ``"row"`` key: } } -Column values can use the :ref:`binary value JSON format ` to write BLOB data. - If successful, this will return a ``201`` status code and the newly inserted row, for example: .. code-block:: json @@ -1860,11 +1727,9 @@ If any of your rows have a primary key that is already in use, you will get an e { "ok": false, - "error": "UNIQUE constraint failed: new_table.id", "errors": [ "UNIQUE constraint failed: new_table.id" - ], - "status": 400 + ] } Pass ``"ignore": true`` to ignore these errors and insert the other rows: @@ -1900,8 +1765,6 @@ An upsert is an insert or update operation. If a row with a matching primary key The upsert API is mostly the same shape as the :ref:`insert API `. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions. -It also accepts the same :ref:`binary value JSON format `. - :: POST //
/-/upsert @@ -1939,7 +1802,7 @@ The above example will: Similar to ``/-/insert``, a ``row`` key with an object can be used instead of a ``rows`` array to upsert a single row. -If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. This is deliberately different from the ``201`` returned by :ref:`insert `: an upsert may update existing rows without creating anything, so it does not claim resource creation. +If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. Add ``"return": true`` to the request body to return full copies of the affected rows after they have been inserted or updated: @@ -1996,11 +1859,9 @@ When using upsert you must provide the primary key column (or columns if the tab { "ok": false, - "error": "Row 0 is missing primary key column(s): \"id\"", "errors": [ "Row 0 is missing primary key column(s): \"id\"" - ], - "status": 400 + ] } If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. @@ -2034,8 +1895,6 @@ To update a row, make a ``POST`` to ``//
//-/update``. You only need to pass the columns you want to update. Any other columns will be left unchanged. -Updated values can use the :ref:`binary value JSON format `. - If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. Add ``"return": true`` to the request body to return the updated row: @@ -2055,16 +1914,14 @@ The returned JSON will look like this: { "ok": true, - "rows": [ - { - "id": 1, - "title": "New title", - "other_column": "Will be present here too" - } - ] + "row": { + "id": 1, + "title": "New title", + "other_column": "Will be present here too" + } } -Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. @@ -2085,7 +1942,7 @@ To delete a row, make a ``POST`` to ``//
//-/delete``. If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. -Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableCreateView: @@ -2241,8 +2098,6 @@ Datasette will create a table with a schema that matches those rows and insert t "pk": "id" } -Example rows can use the :ref:`binary value JSON format `, allowing Datasette to infer ``BLOB`` columns. - Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions. The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: @@ -2267,11 +2122,9 @@ If you pass a row to the create endpoint with a primary key that already exists { "ok": false, - "error": "UNIQUE constraint failed: creatures.id", "errors": [ "UNIQUE constraint failed: creatures.id" - ], - "status": 400 + ] } You can avoid this error by passing the same ``"ignore": true`` or ``"replace": true`` options to the create endpoint as you can to the :ref:`insert endpoint `. @@ -2507,7 +2360,7 @@ A successful response returns the new schema and the previous schema. If the req "operations_applied": 11 } -Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableSetColumnTypeView: @@ -2571,7 +2424,7 @@ To clear an existing column type assignment, set ``column_type`` to ``null``: This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. -Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableDropView: @@ -2608,4 +2461,4 @@ If you pass the following POST body: Then the table will be dropped and a status ``200`` response of ``{"ok": true}`` will be returned. -Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 9a4dba23..422e67f4 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -46,7 +46,7 @@ def table_extras(cog): cog.out("\n") for scope, heading, intro, classes in classes_by_scope: cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) - cog.out(f"{intro}\n\n") + cog.out("{}\n\n".format(intro)) for cls in classes: examples = _examples_for_scope(cls, scope) description = cls.description or "" @@ -58,16 +58,16 @@ def table_extras(cog): if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out(f"``{cls.key()}``\n") - cog.out(f" {description}\n\n") + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) for example in examples: if example.path: value = live_examples[(example.path, example.key or cls.key())] - cog.out(f" ``GET {example.path}``\n\n") + cog.out(" ``GET {}``\n\n".format(example.path)) else: value = example.value if example.note: - cog.out(f" {example.note}\n\n") + cog.out(" {}\n\n".format(example.note)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) cog.out("\n\n") @@ -139,7 +139,7 @@ async def _fetch_live_examples(scoped_classes): response = await datasette.client.get(example.path) assert response.status_code == 200, example.path data = response.json() - assert key in data, f"{key} missing from {example.path}" + assert key in data, "{} missing from {}".format(key, example.path) examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 1bf17f8e..031b3ddd 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,8 +1,7 @@ import json import textwrap - -from ruamel.yaml import YAML from yaml import safe_dump +from ruamel.yaml import YAML def metadata_example(cog, data=None, yaml=None): @@ -34,10 +33,10 @@ def config_example( else: data = input output_yaml = safe_dump(input, sort_keys=False) - cog.out(f"\n.. tab:: {yaml_title}\n\n") + cog.out("\n.. tab:: {}\n\n".format(yaml_title)) cog.out(" .. code-block:: yaml\n\n") cog.out(textwrap.indent(output_yaml, " ")) - cog.out(f"\n\n.. tab:: {json_title}\n\n") + cog.out("\n\n.. tab:: {}\n\n".format(json_title)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") @@ -45,10 +44,8 @@ def config_example( def internal_schema(cog): import asyncio - - from sqlite_utils import Database - from datasette.app import Datasette + from sqlite_utils import Database ds = Datasette() db = ds.get_internal_database() diff --git a/docs/pages.rst b/docs/pages.rst index 65c03a49..ce88be12 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -95,7 +95,7 @@ Use the :ref:`ExecuteWriteView` JSON API to execute writable SQL programmaticall Stored query browsers --------------------- -The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. The JSON versions accept ``?_size=`` (default 50, ``max`` for the :ref:`setting_max_returned_rows` limit) and a ``?_next=`` pagination token. +The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. These pages support search, pagination and filters for read-only or writable queries and private or public queries. Adding a ``.json`` extension to either URL returns the same list as JSON. @@ -169,13 +169,11 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi .. code-block:: json { - "ok": true, "schemas": [ { "database": "content", "schema": "create table posts ..." } - ] } .. _DatabaseSchemaView: @@ -183,11 +181,11 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi Database schema --------------- -Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"`` and ``"schema"`` keys. +Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"`` and ``"schema"`` keys. .. _TableSchemaView: Table schema ------------ -Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"``, ``"table"``, and ``"schema"`` keys. +Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"``, ``"table"``, and ``"schema"`` keys. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 049cb292..81ef4acd 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1685,8 +1685,6 @@ forbidden(datasette, request, message) Plugins can use this to customize how Datasette responds when a 403 Forbidden error occurs - usually because a page failed a permission check, see :ref:`authentication_permissions`. -Datasette's default behavior returns the :ref:`standard JSON error format ` with a 403 status when the request path ends in ``.json`` or the request has an ``Accept: application/json`` or ``Content-Type: application/json`` header; other requests get an HTML error page. - If a plugin hook wishes to react to the error, it should return a :ref:`Response object `. This example returns a redirect to a ``/-/login`` page: @@ -2546,10 +2544,6 @@ The default ``SignedTokenHandler`` uses itsdangerous signed tokens (``dstok_`` p async def verify_token(self, datasette, token): # Look up token in database, return actor dict or None - # if this handler does not recognize the token. Raise - # datasette.TokenInvalid for a token this handler - # recognizes but rejects (revoked, expired) - Datasette - # will respond with a 401 error. ... diff --git a/docs/plugins.rst b/docs/plugins.rst index d32a9fe6..d2b5c20a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -459,8 +459,6 @@ Secret configuration values Some plugins may need configuration that should stay secret - API keys for example. There are two ways in which you can store secret configuration values. -The :ref:`/-/config ` introspection endpoint redacts the values of any configuration keys whose names contain one of these substrings: ``secret``, ``key``, ``password``, ``token``, ``hash`` or ``dsn``. Name your plugin's secret configuration keys accordingly - for example ``api_key`` or ``client_secret`` - so they are automatically redacted there. - **As environment variables**. If your secret lives in an environment variable that is available to the Datasette process, you can indicate that the configuration value should be read from that environment variable like so: .. [[[cog diff --git a/docs/settings.rst b/docs/settings.rst index 9c114e4a..5cd49113 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -125,23 +125,6 @@ You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_insert_rows 1000 -.. _setting_max_post_body_bytes: - -max_post_body_bytes -~~~~~~~~~~~~~~~~~~~ - -Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API `. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB). - -This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over. - -If you increase :ref:`setting_max_insert_rows` to support larger bulk inserts you may need to increase this limit as well:: - - datasette mydatabase.db --setting max_post_body_bytes 10485760 - -Set it to 0 to disable the limit entirely:: - - datasette mydatabase.db --setting max_post_body_bytes 0 - .. _setting_num_sql_threads: num_sql_threads diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 4c6e4426..371348fb 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -657,7 +657,7 @@ There are three options for specifying that you would like the response to your - Include ``?_json=1`` in the URL that you POST to - Include ``"_json": 1`` in your JSON body, or ``&_json=1`` in your form encoded body -A successful JSON response will look like this: +The JSON response will look like this: .. code-block:: json @@ -667,21 +667,7 @@ A successful JSON response will look like this: "redirect": "/data/add_name" } -If the SQL fails to execute - for example a constraint violation - the response uses the :ref:`standard error format ` with a ``400`` status, plus the ``"redirect"`` key from the query configuration: - -.. code-block:: json - - { - "ok": false, - "error": "UNIQUE constraint failed: docs.id", - "errors": [ - "UNIQUE constraint failed: docs.id" - ], - "status": 400, - "redirect": null - } - -The ``"message"``, ``"error"`` and ``"redirect"`` values here take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. +The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. .. _pagination: diff --git a/docs/template_context.rst b/docs/template_context.rst index e445b335..5c6b1567 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -98,7 +98,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re The color assigned to the database ``database_page_data`` - ``dict`` - JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``. + JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``. ``editable`` - ``bool`` Boolean indicating if the database is editable @@ -329,7 +329,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. Pagination token for the next page, or None ``next_url`` - ``str`` - Full URL for the next page of results, or None if there are no more pages. See :ref:`json_api_pagination`. + Full URL for the next page of results ``ok`` - ``bool`` True if the data for this page was retrieved without errors @@ -389,7 +389,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. SQL definition for this table ``table_insert_ui`` - ``dict`` - Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys. + Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys. ``table_page_data`` - ``dict`` JSON data used by JavaScript on the table page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs, optional ``insertRow`` data and optional ``alterTable`` data. diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index 95b8a366..a5f4fb6f 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -21,12 +21,14 @@ def template_context(cog): ), ) for name, doc in TEMPLATE_BASE_CONTEXT.items(): - cog.out(f"``{name}``\n") - cog.out(f" {doc}\n\n") + cog.out("``{}``\n".format(name)) + cog.out(" {}\n\n".format(doc)) for klass in PAGES.values(): title = "{} page".format(klass.__name__.removesuffix("Context")) - intro = f"{klass.__doc__} Rendered using the ``{klass.documented_template}`` template." + intro = "{} Rendered using the ``{}`` template.".format( + klass.__doc__, klass.documented_template + ) _section(cog, title, intro) if klass.extras_scope is not None: cog.out( @@ -34,10 +36,10 @@ def template_context(cog): "` for this page.\n\n" ) for f in sorted(klass.documented_fields(), key=lambda f: f.name): - cog.out(f"``{f.name}`` - ``{f.type_name}``\n") - cog.out(f" {f.help}\n\n") + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) def _section(cog, title, intro): cog.out("{}\n{}\n\n".format(title, "-" * len(title))) - cog.out(f"{intro}\n\n") + cog.out("{}\n\n".format(intro)) diff --git a/pyproject.toml b/pyproject.toml index e658955f..215b2cca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,12 +30,12 @@ dependencies = [ "hupper>=1.9", "httpx>=0.20,<1.0", "pluggy>=1.0", - "uvicorn>=0.29", + "uvicorn>=0.11", "aiofiles>=0.4", "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", - "sqlite-utils>=4.0", + "sqlite-utils>=3.30,<4.0", "asyncinject>=0.7", "setuptools", "pip", @@ -69,7 +69,7 @@ dev = [ "trustme>=0.7", "cogapp>=3.3.0", "multipart-form-data-conformance==0.1a0", - "ruff>=0.16.0", + "ruff>=0.9", # docs "Sphinx==7.4.7", "furo==2025.9.25", @@ -102,5 +102,9 @@ datasette = ["templates/*.html"] [tool.setuptools.dynamic] version = {attr = "datasette.version.__version__"} +[tool.ruff] +line-length = 160 +select = ["E", "F", "W"] + [tool.uv] package = true diff --git a/ruff.toml b/ruff.toml index 3c4345bf..74447a8c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,7 +1,2 @@ line-length = 160 -target-version = "py310" - -[lint.flake8-bugbear] -# from_extra() returns a dataclasses.field(), so it is safe as a dataclass -# default - ruff cannot see through the wrapper (RUF009) -extend-immutable-calls = ["datasette.views.from_extra"] \ No newline at end of file +target-version = "py310" \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 12dce417..7ec03146 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,15 @@ +import httpx import importlib.metadata import os import pathlib +import pytest +import pytest_asyncio import re -import socket import subprocess import sys import tempfile import time from dataclasses import dataclass - -import httpx -import pytest -import pytest_asyncio - from datasette import Event, hookimpl try: @@ -33,29 +30,15 @@ UNDOCUMENTED_PERMISSIONS = { } -def wait_until_responds(url, timeout=5.0, client=httpx, process=None, **kwargs): +def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): start = time.time() while time.time() - start < timeout: - # If the server died there is no point waiting out the timeout - fail - # now, with its output, instead of after `timeout` seconds of silence - if process is not None and process.poll() is not None: - raise AssertionError( - "Server exited early with returncode {}\n{}".format( - process.returncode, process.stdout.read().decode("utf-8") - ) - ) try: client.get(url, **kwargs) return - except httpx.TransportError: + except httpx.ConnectError: time.sleep(0.1) - raise AssertionError(f"Timed out waiting for {url} to respond") - - -def find_free_port(): - with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - return sock.getsockname()[1] + raise AssertionError("Timed out waiting for {} to respond".format(url)) @pytest.fixture @@ -71,12 +54,10 @@ def bare_ds(): @pytest_asyncio.fixture(scope="session") async def ds_client(): - import secrets - from datasette.app import Datasette from datasette.database import Database - from .fixtures import CONFIG, METADATA, PLUGINS_DIR + import secrets ds = Datasette( metadata=METADATA, @@ -115,8 +96,8 @@ def pytest_report_header(config): conn.close() sqlite_utils_version = importlib.metadata.version("sqlite-utils") headers = [ - f"SQLite: {version}", - f"sqlite-utils: {sqlite_utils_version}", + "SQLite: {}".format(version), + "sqlite-utils: {}".format(sqlite_utils_version), ] if config.getoption("--playwright"): try: @@ -194,8 +175,8 @@ def restore_working_directory(tmpdir, request): @pytest.fixture(scope="session", autouse=True) def check_actions_are_documented(): - from datasette.default_actions import register_actions as default_register_actions from datasette.plugins import pm + from datasette.default_actions import register_actions as default_register_actions content = ( pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" @@ -221,7 +202,7 @@ def check_actions_are_documented(): if kwargs["action"] in core_actions: assert ( action in documented_actions - ), f"Undocumented permission action: {action}" + ), "Undocumented permission action: {}".format(action) pm.add_hookcall_monitoring( before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None @@ -316,74 +297,8 @@ def ds_unix_domain_socket_server(tmp_path_factory): pass -@pytest.fixture -def serve_with_plugins(tmp_path): - """Factory fixture for starting ``datasette serve`` in a subprocess with - plugins written to a temporary ``--plugins-dir``. - - For tests that need the real serve path: event-loop wiring, exit codes, - signals. The usual in-process ``pm.register`` plugin pattern can't reach - a subprocess, so plugin source is written out as importable files instead. - - Unlike ``ds_localhost_http_server`` this is function-scoped and takes a - fresh port each time, because each test needs its own plugins. Call it as:: - - proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE}) - - ``plugins`` maps module name to Python source. Pass - ``wait_for_startup=False`` when the server is expected to fail during - startup rather than begin serving. Extra CLI arguments are passed through. - Every process started is terminated when the test ends. - """ - processes = [] - - def start(plugins, *extra_args, wait_for_startup=True): - plugins_dir = tmp_path / "plugins" - plugins_dir.mkdir(exist_ok=True) - for module_name, source in plugins.items(): - (plugins_dir / f"{module_name}.py").write_text(source, "utf-8") - port = find_free_port() - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "datasette", - "--memory", - "--plugins-dir", - str(plugins_dir), - "-h", - "127.0.0.1", - "-p", - str(port), - *extra_args, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - # Avoid FileNotFoundError: [Errno 2] No such file or directory: - cwd=tempfile.gettempdir(), - ) - processes.append(proc) - if wait_for_startup: - wait_until_responds( - f"http://127.0.0.1:{port}/-/versions.json", process=proc - ) - return proc, port - - yield start - - for proc in processes: - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - - # Import fixtures from fixtures.py to make them available -from .fixtures import ( # noqa: F401 - TEMP_PLUGIN_SECRET_FILE, +from .fixtures import ( # noqa: E402, F401 app_client, app_client_base_url_prefix, app_client_conflicting_database_names, @@ -400,4 +315,5 @@ from .fixtures import ( # noqa: F401 app_client_with_dot, app_client_with_trace, make_app_client, + TEMP_PLUGIN_SECRET_FILE, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index 83607c1a..8ab3633f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,13 +1,3 @@ -import contextlib -import json -import os -import pathlib -import tempfile -import textwrap - -import click -import pytest - from datasette.app import Datasette from datasette.fixtures import ( EXTRA_DATABASE_SQL, @@ -15,6 +5,14 @@ from datasette.fixtures import ( write_fixture_database, ) from datasette.utils.testing import TestClient +import click +import contextlib +import json +import os +import pathlib +import pytest +import tempfile +import textwrap # This temp file is used by one of the plugin config tests TEMP_PLUGIN_SECRET_FILE = os.path.join(tempfile.gettempdir(), "plugin-secret") diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index baf20a77..c89ad0a3 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -1,16 +1,16 @@ import asyncio +from datasette import hookimpl +from datasette.facets import Facet +from datasette.tokens import TokenHandler +from datasette import tracer +from datasette.permissions import Action +from datasette.resources import DatabaseResource +from datasette.utils import path_with_added_args +from datasette.utils.asgi import asgi_send_json, Response import base64 import json import urllib.parse -from datasette import hookimpl, tracer -from datasette.facets import Facet -from datasette.permissions import Action -from datasette.resources import DatabaseResource -from datasette.tokens import TokenHandler -from datasette.utils import path_with_added_args -from datasette.utils.asgi import Response, asgi_send_json - @hookimpl def prepare_connection(conn, database, datasette): @@ -305,7 +305,11 @@ def startup(datasette): datasette._startup_hook_fired = True # And test some import shortcuts too - from datasette import Forbidden, NotFound, Response, actor_matches_allow, hookimpl + from datasette import Response + from datasette import Forbidden + from datasette import NotFound + from datasette import hookimpl + from datasette import actor_matches_allow _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow) @@ -369,7 +373,7 @@ def table_actions(datasette, database, table, actor, request): "label": "Plugin button", "description": "Runs JavaScript from a plugin", "attrs": { - "aria-label": f"Plugin button for {table}", + "aria-label": "Plugin button for {}".format(table), "data-plugin-action": "plugin-button", "data-database": database, "data-table": table, diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 26e45a5b..864637a6 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -1,10 +1,8 @@ -import json -from functools import wraps - -import markupsafe - from datasette import hookimpl from datasette.utils.asgi import Response +from functools import wraps +import markupsafe +import json @hookimpl @@ -35,7 +33,11 @@ def render_cell(value, database): if set(data.keys()) != {"href", "label"}: return None href = data["href"] - if not (href.startswith(("/", "http://", "https://"))): + if not ( + href.startswith("/") + or href.startswith("http://") + or href.startswith("https://") + ): return None return markupsafe.Markup( '{label}'.format( @@ -52,7 +54,7 @@ def extra_template_vars(template, database, table, view_name, request, datasette datasette._last_request = request async def query_database(sql): - first_db = next(iter(datasette.databases.keys())) + first_db = list(datasette.databases.keys())[0] return (await datasette.execute(first_db, sql)).rows[0][0] async def inner(): @@ -170,10 +172,10 @@ def register_routes(datasette): path = config["path"] def new_table(request): - return Response.text(f"/db/table: {sorted(request.url_vars.items())}") + return Response.text("/db/table: {}".format(sorted(request.url_vars.items()))) return [ - (rf"/{path}/$", lambda: Response.text(path.upper())), + (r"/{}/$".format(path), lambda: Response.text(path.upper())), # Also serves to demonstrate over-ride of default paths: (r"/(?P[^/]+)/(?P[^/]+?$)", new_table), ] diff --git a/tests/plugins/register_output_renderer.py b/tests/plugins/register_output_renderer.py index 671f2d1e..cfe15215 100644 --- a/tests/plugins/register_output_renderer.py +++ b/tests/plugins/register_output_renderer.py @@ -1,7 +1,6 @@ -import json - from datasette import hookimpl from datasette.utils.asgi import Response +import json async def can_render( @@ -19,7 +18,9 @@ async def can_render( "request": request, "view_name": view_name, } - return not request.args.get("_no_can_render") + if request.args.get("_no_can_render"): + return False + return True async def render_test_all_parameters( diff --git a/tests/plugins/sleep_sql_function.py b/tests/plugins/sleep_sql_function.py index 2fca1d66..d4b32a09 100644 --- a/tests/plugins/sleep_sql_function.py +++ b/tests/plugins/sleep_sql_function.py @@ -1,6 +1,5 @@ -import time - from datasette import hookimpl +import time @hookimpl diff --git a/tests/test_actions_sql.py b/tests/test_actions_sql.py index 76320bd1..a1fca971 100644 --- a/tests/test_actions_sql.py +++ b/tests/test_actions_sql.py @@ -10,11 +10,10 @@ These tests verify: import pytest import pytest_asyncio - -from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.resources import DatabaseResource, QueryResource, TableResource +from datasette import hookimpl def test_resource_string_representations(): @@ -91,7 +90,7 @@ async def test_allowed_resources_global_allow(test_ds): assert all(isinstance(t, TableResource) for t in tables) # Check specific tables are present - table_set = {(t.parent, t.child) for t in tables} + table_set = set((t.parent, t.child) for t in tables) assert ("analytics", "events") in table_set assert ("analytics", "users") in table_set assert ("analytics", "sensitive") in table_set diff --git a/tests/test_actor_restriction_bug.py b/tests/test_actor_restriction_bug.py index 6e633ff6..0bfc9e1e 100644 --- a/tests/test_actor_restriction_bug.py +++ b/tests/test_actor_restriction_bug.py @@ -6,7 +6,6 @@ config allow blocks can bypass table-level restrictions. """ import pytest - from datasette.app import Datasette from datasette.resources import TableResource diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 2f20e8e8..08b952fb 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -10,8 +10,6 @@ Layer 3: table/database views precompute all registered actions before import pytest import pytest_asyncio - -from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import ( Action, @@ -20,6 +18,7 @@ from datasette.permissions import ( _permission_check_cache, ) from datasette.resources import DatabaseResource, TableResource +from datasette import hookimpl class CountingRulesPlugin: @@ -115,7 +114,7 @@ async def test_allowed_not_memoized_without_cache(counting_ds): async def test_cache_keyed_on_full_actor_identity(counting_ds): """Interleaved checks for different actors never share cache entries.""" # Uses drop-table because default permissions deny it to non-root actors - ds, _plugin = counting_ds + ds, plugin = counting_ds resource = TableResource("analytics", "users") token = _permission_check_cache.set({}) try: @@ -181,7 +180,7 @@ async def test_cache_keyed_on_resource(counting_ds): @pytest.mark.asyncio async def test_skip_permission_checks_bypasses_cache(counting_ds): - ds, _plugin = counting_ds + ds, plugin = counting_ds resource = TableResource("analytics", "users") token = _permission_check_cache.set({}) try: diff --git a/tests/test_allowed_resources.py b/tests/test_allowed_resources.py index e251deab..e247aa78 100644 --- a/tests/test_allowed_resources.py +++ b/tests/test_allowed_resources.py @@ -7,10 +7,9 @@ based on permission rules from plugins and configuration. import pytest import pytest_asyncio - -from datasette import hookimpl from datasette.app import Datasette from datasette.permissions import PermissionSQL +from datasette import hookimpl # Test plugin that provides permission rules diff --git a/tests/test_api.py b/tests/test_api.py index 76c30e46..f57d0206 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,15 +1,12 @@ -import pathlib -import urllib - -import pytest - from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode from datasette.utils.sqlite import sqlite_version from datasette.version import __version__ - -from .fixtures import EXPECTED_PLUGINS, make_app_client +from .fixtures import make_app_client, EXPECTED_PLUGINS +import pathlib +import pytest +import sys +import urllib @pytest.mark.asyncio @@ -18,7 +15,7 @@ async def test_homepage(ds_client): assert response.status_code == 200 assert "application/json; charset=utf-8" == response.headers["content-type"] data = response.json() - assert sorted(data.get("metadata").keys()) == [ + assert sorted(list(data.get("metadata").keys())) == [ "about", "about_url", "description_html", @@ -29,9 +26,8 @@ async def test_homepage(ds_client): "title", ] databases = data.get("databases") - assert isinstance(databases, list) - assert [d["name"] for d in databases] == ["fixtures"] - d = databases[0] + assert databases.keys() == {"fixtures": 0}.keys() + d = databases["fixtures"] assert d["name"] == "fixtures" assert isinstance(d["tables_count"], int) assert isinstance(len(d["tables_and_views_truncated"]), int) @@ -46,7 +42,8 @@ async def test_homepage_sort_by_relationships(ds_client): response = await ds_client.get("/.json?_sort=relationships") assert response.status_code == 200 tables = [ - t["name"] for t in response.json()["databases"][0]["tables_and_views_truncated"] + t["name"] + for t in response.json()["databases"]["fixtures"]["tables_and_views_truncated"] ] assert tables == [ "simple_primary_key", @@ -253,10 +250,8 @@ def test_no_files_uses_memory_database(app_client_no_files): response = app_client_no_files.get("/.json") assert response.status == 200 assert { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, - "databases": [ - { + "databases": { + "_memory": { "name": "_memory", "hash": None, "color": "a6c7b9", @@ -271,7 +266,7 @@ def test_no_files_uses_memory_database(app_client_no_files): "views_count": 0, "private": False, }, - ], + }, "metadata": {}, } == response.json # Try that SQL query @@ -328,15 +323,20 @@ def test_sql_time_limit(app_client_shorter_time_limit): "/fixtures/-/query.json?sql=select+sleep(0.5)", ) assert 400 == response.status - expected_message = ( - "SQL query took too long. The time limit is" - " controlled by the sql_time_limit_ms setting." - ) assert response.json == { "ok": False, - "error": expected_message, - "errors": [expected_message], + "error": ( + "

SQL query took too long. The time limit is controlled by the\n" + 'sql_time_limit_ms\n' + "configuration option.

\n" + '\n' + "" + ), "status": 400, + "title": "SQL Interrupted", } @@ -350,7 +350,7 @@ async def test_custom_sql_time_limit(ds_client): "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5", ) assert response.status_code == 400 - assert response.json()["error"].startswith("SQL query took too long.") + assert response.json()["title"] == "SQL Interrupted" @pytest.mark.asyncio @@ -371,38 +371,6 @@ async def test_row(ds_client): assert response.json()["rows"] == [{"id": 1, "content": "hello"}] -@pytest.mark.asyncio -@pytest.mark.parametrize("suffix", ("", ".json")) -@pytest.mark.parametrize( - "row_path", - ( - "a", # too few components for a two-column primary key - "a,b,c", # too many components for a two-column primary key - ), -) -async def test_row_pk_arity_mismatch_returns_400(ds_client, row_path, suffix): - # A row URL with the wrong number of comma-separated primary key - # components used to raise an uncaught sqlite3.ProgrammingError (HTTP 500) - # because the SQL had one bind placeholder per PK column but params were - # only bound for the supplied components. It should be a 400 instead, - # mirroring the existing guard in datasette/views/table.py. - response = await ds_client.get(f"/fixtures/compound_primary_key/{row_path}{suffix}") - assert response.status_code == 400 - if suffix == ".json": - assert response.json()["ok"] is False - assert response.json()["status"] == 400 - - -@pytest.mark.asyncio -async def test_row_compound_pk_correct_arity(ds_client): - # The valid two-component URL still resolves the row. - response = await ds_client.get( - "/fixtures/compound_primary_key/a,b.json?_shape=objects" - ) - assert response.status_code == 200 - assert response.json()["rows"] == [{"pk1": "a", "pk2": "b", "content": "c"}] - - @pytest.mark.asyncio async def test_row_strange_table_name(ds_client): response = await ds_client.get( @@ -461,7 +429,7 @@ async def test_row_foreign_key_tables(ds_client): @pytest.mark.asyncio async def test_row_extras(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables,column_details" + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" ) assert response.status_code == 200 data = response.json() @@ -478,45 +446,6 @@ async def test_row_extras(ds_client): "format": "json", } assert len(data["foreign_key_tables"]) == 5 - id_detail = data["column_details"]["id"] - assert id_detail["type"].lower() == "integer" - assert id_detail == { - "type": id_detail["type"], - "sqlite_type": "INTEGER", - "notnull": False, - "default": None, - "is_pk": True, - "pk_position": 1, - "hidden": 0, - } - content_detail = data["column_details"]["content"] - assert content_detail["type"].lower() == "text" - assert content_detail == { - "type": content_detail["type"], - "sqlite_type": "TEXT", - "notnull": False, - "default": None, - "is_pk": False, - "pk_position": 0, - "hidden": 0, - } - - -@pytest.mark.asyncio -async def test_column_details_extra_row_for_null_blob(ds_client): - response = await ds_client.get("/fixtures/binary_data/3.json?_extra=column_details") - assert response.status_code == 200 - data_detail = response.json()["column_details"]["data"] - assert data_detail["type"].lower() == "blob" - assert data_detail == { - "type": data_detail["type"], - "sqlite_type": "BLOB", - "notnull": False, - "default": None, - "is_pk": False, - "pk_position": 0, - "hidden": 0, - } @pytest.mark.asyncio @@ -578,7 +507,7 @@ async def test_row_extra_render_cell(): def test_databases_json(app_client_two_attached_databases_one_immutable): response = app_client_two_attached_databases_one_immutable.get("/-/databases.json") - databases = response.json["databases"] + databases = response.json assert 2 == len(databases) extra_database, fixtures_database = databases assert "extra database" == extra_database["name"] @@ -594,13 +523,10 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): @pytest.mark.asyncio async def test_threads_json(ds_client): - ds_client.ds.root_enabled = True - try: - response = await ds_client.get("/-/threads.json", actor={"id": "root"}) - finally: - ds_client.ds.root_enabled = False - expected_keys = {"ok", "threads", "num_threads"} - expected_keys.update({"tasks", "num_tasks"}) + response = await ds_client.get("/-/threads.json") + expected_keys = {"threads", "num_threads"} + if sys.version_info >= (3, 7, 0): + expected_keys.update({"tasks", "num_tasks"}) data = response.json() assert set(data.keys()) == expected_keys # Should be at least one _execute_writes thread for __INTERNAL__ @@ -651,7 +577,7 @@ async def test_actions_json(ds_client): try: ds_client.ds.root_enabled = True response = await ds_client.get("/-/actions.json", actor={"id": "root"}) - data = response.json()["actions"] + data = response.json() finally: ds_client.ds.root_enabled = original_root_enabled assert isinstance(data, list) @@ -683,7 +609,6 @@ async def test_actions_json(ds_client): async def test_settings_json(ds_client): response = await ds_client.get("/-/settings.json") assert response.json() == { - "ok": True, "default_page_size": 50, "default_facet_size": 30, "default_allow_sql": True, @@ -691,7 +616,6 @@ async def test_settings_json(ds_client): "facet_time_limit_ms": 200, "max_returned_rows": 100, "max_insert_rows": 100, - "max_post_body_bytes": 2 * 1024 * 1024, "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, @@ -753,7 +677,7 @@ def test_config_cache_size(app_client_larger_cache_size): def test_config_force_https_urls(): with make_app_client(settings={"force_https_urls": True}) as client: response = client.get( - "/fixtures/facetable.json?_size=3&_facet=state&_extra=suggested_facets" + "/fixtures/facetable.json?_size=3&_facet=state&_extra=next_url,suggested_facets" ) assert response.json["next_url"].startswith("https://") assert response.json["facet_results"]["results"]["state"]["results"][0][ @@ -848,9 +772,7 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 assert ["foo-bar", "foo", "fixtures"] == [ d["name"] - for d in app_client_conflicting_database_names.get("/-/databases.json").json[ - "databases" - ] + for d in app_client_conflicting_database_names.get("/-/databases.json").json ] for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): data = app_client_conflicting_database_names.get(path).json @@ -921,41 +843,13 @@ async def test_tilde_encoded_database_names(db_name): ds = Datasette() ds.add_memory_database(db_name) response = await ds.client.get("/.json") - databases_by_name = {d["name"]: d for d in response.json()["databases"]} - assert db_name in databases_by_name - path = databases_by_name[db_name]["path"] + assert db_name in response.json()["databases"].keys() + path = response.json()["databases"][db_name]["path"] # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 -@pytest.mark.asyncio -@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar")) -async def test_table_with_reserved_characters_in_name(table_name): - # Table names containing characters such as "]" that cannot be escaped - # using SQLite [bracket] quoting used to break schema introspection and - # the table page - https://github.com/simonw/datasette/issues/2431 - ds = Datasette() - db = ds.add_memory_database("test_reserved_table_names") - await db.execute_write( - f"create table {escape_sqlite(table_name)} (id integer primary key, name text)" - ) - await db.execute_write( - f"insert into {escape_sqlite(table_name)} (id, name) values (1, 'one')" - ) - # Schema introspection (populate_schema_tables) must not crash: - db_response = await ds.client.get("/test_reserved_table_names.json") - assert db_response.status_code == 200 - tables = {t["name"]: t for t in db_response.json()["tables"]} - assert tables[table_name]["count"] == 1 - # And the table page itself must load and return the row: - table_response = await ds.client.get( - f"/test_reserved_table_names/{tilde_encode(table_name)}.json?_shape=array" - ) - assert table_response.status_code == 200 - assert table_response.json() == [{"id": 1, "name": "one"}] - - @pytest.mark.asyncio @pytest.mark.parametrize( "config,expected", @@ -989,7 +883,7 @@ async def test_config_json(config, expected): "/-/config.json should return redacted configuration" ds = Datasette(config=config) response = await ds.client.get("/-/config.json") - assert response.json() == {"ok": True, **expected} + assert response.json() == expected @pytest.mark.asyncio @@ -1085,7 +979,7 @@ async def test_config_json(config, expected): async def test_upgrade_metadata(metadata, expected_config, expected_metadata): ds = Datasette(metadata=metadata) response = await ds.client.get("/-/config.json") - assert response.json() == {"ok": True, **expected_config} + assert response.json() == expected_config response2 = await ds.client.get("/-/metadata.json") assert response2.json() == expected_metadata diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 11ef30de..563ca21e 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,25 +1,9 @@ -import time - -import pytest -import sqlite_utils - from datasette.app import Datasette from datasette.events import RenameTableEvent -from datasette.utils import error_body, escape_sqlite, sqlite3 - +from datasette.utils import escape_sqlite, sqlite3 from .utils import last_event - - -def assert_schema_contains(fragment, schema): - assert ( - fragment in schema - ), f"Expected schema to contain {fragment!r}, got {schema!r}" - - -def assert_schema_not_contains(fragment, schema): - assert ( - fragment not in schema - ), f"Expected schema not to contain {fragment!r}, got {schema!r}" +import pytest +import time @pytest.fixture @@ -51,7 +35,7 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", } @@ -59,141 +43,13 @@ def _headers(token): def _insert_and_fetch_created(conn, table, insert_sql): cursor = conn.execute(insert_sql) return conn.execute( - f"select created, typeof(created) from {escape_sqlite(table)} where rowid = ?", + "select created, typeof(created) from {} where rowid = ?".format( + escape_sqlite(table) + ), (cursor.lastrowid,), ).fetchone() -BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"} -BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}' - - -@pytest.mark.asyncio -async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write): - token = write_token(ds_write) - response = await ds_write.client.post( - "/data/-/create", - json={ - "table": "binary_create", - "row": { - "id": 1, - "data": BASE64_WRITE_API_VALUE, - "literal": {"$raw": BASE64_WRITE_API_VALUE}, - "double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, - }, - "pk": "id", - }, - headers=_headers(token), - ) - assert response.status_code == 201 - assert_schema_contains('"data" BLOB', response.json()["schema"]) - assert_schema_contains('"literal" TEXT', response.json()["schema"]) - - rows = (await ds_write.get_database("data").execute(""" - select - typeof(data) as data_type, - hex(data) as data_hex, - typeof(literal) as literal_type, - literal, - typeof(double_raw) as double_raw_type, - double_raw - from binary_create - """)).dicts() - assert rows == [ - { - "data_type": "blob", - "data_hex": "000102FDFEFF", - "literal_type": "text", - "literal": BASE64_WRITE_API_LITERAL, - "double_raw_type": "text", - "double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', - } - ] - - -@pytest.mark.asyncio -async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write): - token = write_token(ds_write) - db = ds_write.get_database("data") - await db.execute_write( - "create table binary_api (id integer primary key, data blob, literal text)" - ) - - insert_response = await ds_write.client.post( - "/data/binary_api/-/insert", - json={ - "row": { - "id": 1, - "data": BASE64_WRITE_API_VALUE, - "literal": {"$raw": BASE64_WRITE_API_VALUE}, - } - }, - headers=_headers(token), - ) - assert insert_response.status_code == 201 - assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE - - upsert_response = await ds_write.client.post( - "/data/binary_api/-/upsert", - json={ - "rows": [ - { - "id": 2, - "data": BASE64_WRITE_API_VALUE, - "literal": {"$raw": BASE64_WRITE_API_VALUE}, - } - ] - }, - headers=_headers(token), - ) - assert upsert_response.status_code == 200 - assert upsert_response.json() == {"ok": True} - - update_response = await ds_write.client.post( - "/data/binary_api/1/-/update", - json={ - "update": { - "data": {"$base64": True, "encoded": "/wAB"}, - "literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, - }, - "return": True, - }, - headers=_headers(token), - ) - assert update_response.status_code == 200 - assert update_response.json()["rows"][0]["data"] == { - "$base64": True, - "encoded": "/wAB", - } - - rows = (await db.execute(""" - select - id, - typeof(data) as data_type, - hex(data) as data_hex, - typeof(literal) as literal_type, - literal - from binary_api - order by id - """)).dicts() - assert rows == [ - { - "id": 1, - "data_type": "blob", - "data_hex": "FF0001", - "literal_type": "text", - "literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', - }, - { - "id": 2, - "data_type": "blob", - "data_hex": "000102FDFEFF", - "literal_type": "text", - "literal": BASE64_WRITE_API_LITERAL, - }, - ] - - @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) @@ -243,7 +99,7 @@ async def test_insert_row(ds_write, content_type): "/data/docs/-/insert", json={"row": {"title": "Test", "score": 1.2, "age": 5}}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": content_type, }, ) @@ -288,7 +144,11 @@ async def test_insert_row_alter(ds_write): @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): token = write_token(ds_write) - data = {"rows": [{"title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20)]} + data = { + "rows": [ + {"title": "Test {}".format(i), "score": 1.0, "age": 5} for i in range(20) + ] + } if return_rows: data["return"] = True response = await ds_write.client.post( @@ -312,41 +172,14 @@ async def test_insert_rows(ds_write, return_rows): ).dicts() assert len(actual_rows) == 20 assert actual_rows == [ - {"id": i + 1, "title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20) + {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} + for i in range(20) ] assert response.json()["ok"] is True if return_rows: assert response.json()["rows"] == actual_rows -@pytest.mark.asyncio -async def test_insert_rows_post_body_too_large(tmp_path_factory): - db_path = str(tmp_path_factory.mktemp("dbs") / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("create table docs (id integer primary key, title text)") - conn.close() - ds = Datasette([db_path], settings={"max_post_body_bytes": 100}) - ds.root_enabled = True - token = write_token(ds) - response = await ds.client.post( - "/data/docs/-/insert", - json={"rows": [{"title": "x" * 200}]}, - headers=_headers(token), - ) - assert response.status_code == 413 - assert response.json() == error_body( - ["Request body exceeded maximum size of 100 bytes"], 413 - ) - # A small body should still work - response2 = await ds.client.post( - "/data/docs/-/insert", - json={"row": {"title": "hi"}}, - headers=_headers(token), - ) - assert response2.status_code == 201 - ds.close() - - @pytest.mark.asyncio @pytest.mark.parametrize( "path,input,special_case,expected_status,expected_errors", @@ -369,8 +202,8 @@ async def test_insert_rows_post_body_too_large(tmp_path_factory): "/data/docs/-/insert", {"rows": [{"title": "Test"} for i in range(10)]}, "bad_token", - 401, - ["Invalid token signature"], + 403, + ["Permission denied"], ), ( "/data/docs/-/insert", @@ -381,6 +214,13 @@ async def test_insert_rows_post_body_too_large(tmp_path_factory): "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ], ), + ( + "/data/docs/-/insert", + {}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), ( "/data/docs/-/insert", [], @@ -558,21 +398,24 @@ async def test_insert_or_upsert_row_errors( ) if special_case == "bad_token": token += "bad" - kwargs = { - "json": input, - "headers": { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", + kwargs = dict( + json=input, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), }, - } + ) - if special_case != "bad_token": - actor_response = ( - await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) - ).json() - assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( - token_permissions - ) + actor_response = ( + await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) + ).json() + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -619,7 +462,7 @@ async def test_upsert_permissions_per_table(ds_write, allowed): "/data/docs/-/upsert", json={"rows": [{"id": 1, "title": "One"}]}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), }, ) if allowed: @@ -856,7 +699,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Should be a single row assert ( await ds_write.client.get( - f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}" + "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( + table + ) ) ).json() == [1] # Now delete the row @@ -864,12 +709,14 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Special case for that rowid table delete_path = ( await ds_write.client.get( - f"/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{table}" + "/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}".format( + table + ) ) ).json()[0] delete_response = await ds_write.client.post( - f"/data/{table}/{delete_path}/-/delete", + "/data/{}/{}/-/delete".format(table, delete_path), headers=_headers(write_token(ds_write)), ) assert delete_response.status_code == 200 @@ -882,7 +729,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( - f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}" + "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( + table + ) ) ).json() == [0] @@ -932,26 +781,21 @@ async def test_update_row_invalid_key(ds_write): pk = await _insert_row(ds_write) - path = f"/data/docs/{pk}/-/update" + path = "/data/docs/{}/-/update".format(pk) response = await ds_write.client.post( path, json={"update": {"title": "New title"}, "bad_key": 1}, headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "error": "Invalid keys: bad_key", - "errors": ["Invalid keys: bad_key"], - "status": 400, - } + assert response.json() == {"ok": False, "errors": ["Invalid keys: bad_key"]} @pytest.mark.asyncio async def test_update_row_alter(ds_write): token = write_token(ds_write, permissions=["ur", "at"]) pk = await _insert_row(ds_write) - path = f"/data/docs/{pk}/-/update" + path = "/data/docs/{}/-/update".format(pk) response = await ds_write.client.post( path, json={"update": {"title": "New title", "extra": "extra"}, "alter": True}, @@ -1065,6 +909,43 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "table,pk_columns", + ( + ("alter_single_pk", ["id"]), + ("alter_compound_pk", ["tenant", "id"]), + ), +) +async def test_alter_table_primary_keys_are_not_null(ds_write, table, pk_columns): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write( + "create table {} (id integer, tenant text, title text)".format( + escape_sqlite(table) + ) + ) + + response = await ds_write.client.post( + "/data/{}/-/alter".format(table), + json={ + "operations": [ + {"op": "set_primary_key", "args": {"columns": pk_columns}}, + ] + }, + headers=_headers(token), + ) + + assert response.status_code == 200, response.text + columns = ( + await db.execute( + "select * from pragma_table_info(?) where pk > 0 order by pk", [table] + ) + ).dicts() + assert [column["name"] for column in columns] == pk_columns + assert [column["notnull"] for column in columns] == [1] * len(pk_columns) + + @pytest.mark.asyncio @pytest.mark.parametrize( "default_expr,minimum_value,expected_schema", @@ -1107,9 +988,9 @@ async def test_alter_table_integer_default_expr( assert expected_schema in data["schema"] columns = await db.execute("select * from pragma_table_info('docs')") - created_column = next( + created_column = [ column for column in columns.dicts() if column["name"] == "created" - ) + ][0] assert created_column["type"] == "INTEGER" assert expected_schema in created_column["dflt_value"] @@ -1197,9 +1078,7 @@ async def test_alter_table_foreign_key_operations(ds_write): assert response.status_code == 200, response.text data = response.json() assert data["operations_applied"] == 2 - assert_schema_contains( - '"owner_id" INTEGER REFERENCES "owners"("id")', data["schema"] - ) + assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] response = await ds_write.client.post( "/data/docs/-/alter", @@ -1210,7 +1089,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert_schema_not_contains('"owner_id" INTEGER REFERENCES', data["schema"]) + assert "[owner_id] INTEGER REFERENCES" not in data["schema"] response = await ds_write.client.post( "/data/docs/-/alter", @@ -1234,9 +1113,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert_schema_contains( - '"owner_id" INTEGER REFERENCES "categories"("id")', data["schema"] - ) + assert "[owner_id] INTEGER REFERENCES [categories]([id])" in data["schema"] response = await ds_write.client.post( "/data/docs/-/alter", @@ -1245,7 +1122,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert_schema_not_contains('"owner_id" INTEGER REFERENCES', data["schema"]) + assert "[owner_id] INTEGER REFERENCES" not in data["schema"] @pytest.mark.asyncio @@ -1263,9 +1140,10 @@ async def test_alter_table_foreign_key_requires_fk_table_for_fk_column(ds_write) headers=_headers(write_token(ds_write, permissions=["at"])), ) assert response.status_code == 400 - assert response.json() == error_body( - ["operations.0.add_foreign_key.args: fk_column requires fk_table"], 400 - ) + assert response.json() == { + "ok": False, + "errors": ["operations.0.add_foreign_key.args: fk_column requires fk_table"], + } @pytest.mark.asyncio @@ -1289,9 +1167,10 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == error_body( - ["Could not detect single primary key for table 'accounts'"], 400 - ) + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'accounts'"], + } @pytest.mark.asyncio @@ -1357,7 +1236,10 @@ async def test_foreign_key_suggestions_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == error_body(["Permission denied: need alter-table"], 403) + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } @pytest.mark.asyncio @@ -1410,8 +1292,7 @@ async def test_foreign_key_targets(ds_write): await db.execute_write("create table no_pk (name text)") try: await db.execute_write("create virtual table search_docs using fts5(body)") - except Exception: # noqa: BLE001, S110 - # FTS5 is not available in every SQLite build + except Exception: pass response = await ds_write.client.get( @@ -1469,7 +1350,10 @@ async def test_foreign_key_targets_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == error_body(["Permission denied: need create-table"], 403) + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need create-table"], + } @pytest.mark.asyncio @@ -1492,7 +1376,10 @@ async def test_alter_table_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == error_body(["Permission denied: need alter-table"], 403) + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } @pytest.mark.asyncio @@ -1617,7 +1504,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return): token = write_token(ds_write) pk = await _insert_row(ds_write) - path = f"/data/docs/{pk}/-/update" + path = "/data/docs/{}/-/update".format(pk) data = {"update": input} if use_return: @@ -1636,9 +1523,9 @@ async def test_update_row(ds_write, input, expected_errors, use_return): assert response.json()["ok"] is True if not use_return: - assert "rows" not in response.json() + assert "row" not in response.json() else: - returned_row = response.json()["rows"][0] + returned_row = response.json()["row"] assert returned_row["id"] == pk for k, v in input.items(): assert returned_row[k] == v @@ -1652,7 +1539,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return): # And fetch the row to check it's updated response = await ds_write.client.get( - f"/data/docs/{pk}.json?_shape=array", + "/data/docs/{}.json?_shape=array".format(pk), ) assert response.status_code == 200 row = response.json()[0] @@ -1726,42 +1613,6 @@ async def test_drop_table(ds_write, scenario): assert (await ds_write.client.get("/data/docs")).status_code == 404 -@pytest.mark.asyncio -async def test_drop_table_cleans_up_fts(ds_write): - db = ds_write.get_database("data") - - def enable_fts(conn): - sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True) - - await db.execute_write_fn(enable_fts) - assert { - row[0] - for row in await db.execute( - "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" - ) - } == { - "docs_fts", - "docs_fts_config", - "docs_fts_data", - "docs_fts_docsize", - "docs_fts_idx", - } - - response = await ds_write.client.post( - "/data/docs/-/drop", - json={"confirm": True}, - headers=_headers(write_token(ds_write)), - ) - - assert response.json() == {"ok": True} - assert [ - row[0] - for row in await db.execute( - "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" - ) - ] == [] - - @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_status,expected_response,expected_events", @@ -1809,12 +1660,12 @@ async def test_drop_table_cleans_up_fts(ds_write): "table_url": "http://localhost/data/one", "table_api_url": "http://localhost/data/one.json", "schema": ( - 'CREATE TABLE "one" (\n' - ' "id" INTEGER PRIMARY KEY,\n' - ' "title" TEXT,\n' - ' "score" INTEGER,\n' - ' "weight" REAL,\n' - ' "thumbnail" BLOB\n' + "CREATE TABLE [one] (\n" + " [id] INTEGER PRIMARY KEY NOT NULL,\n" + " [title] TEXT,\n" + " [score] INTEGER,\n" + " [weight] FLOAT,\n" + " [thumbnail] BLOB\n" ")" ), }, @@ -1846,10 +1697,10 @@ async def test_drop_table_cleans_up_fts(ds_write): "table_url": "http://localhost/data/two", "table_api_url": "http://localhost/data/two.json", "schema": ( - 'CREATE TABLE "two" (\n' - ' "id" INTEGER PRIMARY KEY,\n' - ' "title" TEXT,\n' - ' "score" REAL\n' + "CREATE TABLE [two] (\n" + " [id] INTEGER PRIMARY KEY NOT NULL,\n" + " [title] TEXT,\n" + " [score] FLOAT\n" ")" ), "row_count": 2, @@ -1875,10 +1726,10 @@ async def test_drop_table_cleans_up_fts(ds_write): "table_url": "http://localhost/data/three", "table_api_url": "http://localhost/data/three.json", "schema": ( - 'CREATE TABLE "three" (\n' - ' "id" INTEGER PRIMARY KEY,\n' - ' "title" TEXT,\n' - ' "score" REAL\n' + "CREATE TABLE [three] (\n" + " [id] INTEGER PRIMARY KEY NOT NULL,\n" + " [title] TEXT,\n" + " [score] FLOAT\n" ")" ), "row_count": 1, @@ -1900,7 +1751,7 @@ async def test_drop_table_cleans_up_fts(ds_write): "table": "four", "table_url": "http://localhost/data/four", "table_api_url": "http://localhost/data/four.json", - "schema": ('CREATE TABLE "four" (\n' ' "name" TEXT\n' ")"), + "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), "row_count": 1, }, ["create-table", "insert-rows"], @@ -1920,8 +1771,9 @@ async def test_drop_table_cleans_up_fts(ds_write): "table_url": "http://localhost/data/five", "table_api_url": "http://localhost/data/five.json", "schema": ( - 'CREATE TABLE "five" (\n "type" TEXT,\n "key" INTEGER,\n' - ' "title" TEXT,\n PRIMARY KEY ("type", "key")\n)' + "CREATE TABLE [five] (\n [type] TEXT NOT NULL,\n" + " [key] INTEGER NOT NULL,\n [title] TEXT,\n" + " PRIMARY KEY ([type], [key])\n)" ), "row_count": 1, }, @@ -2207,18 +2059,87 @@ async def test_create_table( ) assert response.status_code == expected_status data = response.json() - if expected_response.get("ok") is False: - # Error expectations list their messages; derive the canonical envelope - expected_response = error_body(expected_response["errors"], expected_status) - if isinstance(expected_response, dict) and "schema" in expected_response: - assert data.get("schema") == expected_response["schema"] - expected_response = dict(expected_response, schema=data.get("schema")) assert data == expected_response # Should have tracked the expected events events = ds_write._tracked_events assert [e.name for e in events] == expected_events +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,pk_columns", + ( + ( + { + "table": "pk_from_columns", + "columns": [ + {"name": "id", "type": "integer"}, + {"name": "title", "type": "text"}, + ], + "pk": "id", + }, + ["id"], + ), + ( + { + "table": "compound_pk_from_columns", + "columns": [ + {"name": "tenant", "type": "text"}, + {"name": "id", "type": "integer"}, + {"name": "title", "type": "text"}, + ], + "pks": ["tenant", "id"], + }, + ["tenant", "id"], + ), + ( + { + "table": "pk_omitted_from_columns", + "columns": [ + {"name": "title", "type": "text"}, + ], + "pk": "id", + }, + ["id"], + ), + ( + { + "table": "pk_from_rows", + "rows": [{"id": 1, "title": "Row 1"}], + "pk": "id", + }, + ["id"], + ), + ( + { + "table": "compound_pk_from_rows", + "row": {"tenant": "datasette", "id": 1, "title": "Row 1"}, + "pks": ["tenant", "id"], + }, + ["tenant", "id"], + ), + ), +) +async def test_create_table_primary_keys_are_not_null(ds_write, body, pk_columns): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json=body, + headers=_headers(token), + ) + + assert response.status_code == 201, response.text + db = ds_write.get_database("data") + columns = ( + await db.execute( + "select * from pragma_table_info(?) where pk > 0 order by pk", + [body["table"]], + ) + ).dicts() + assert [column["name"] for column in columns] == pk_columns + assert [column["notnull"] for column in columns] == [1] * len(pk_columns) + + @pytest.mark.asyncio async def test_create_table_with_foreign_key(ds_write): token = write_token(ds_write) @@ -2255,9 +2176,7 @@ async def test_create_table_with_foreign_key(ds_write): ) assert response.status_code == 201 data = response.json() - assert_schema_contains( - '"owner_id" INTEGER REFERENCES "owners"("id")', data["schema"] - ) + assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] @pytest.mark.asyncio @@ -2334,7 +2253,7 @@ async def test_create_table_integer_default_expr( ds_write, default_expr, minimum_value, expected_schema ): token = write_token(ds_write) - table = f"default_{default_expr}" + table = "default_{}".format(default_expr) response = await ds_write.client.post( "/data/-/create", json={ @@ -2362,7 +2281,7 @@ async def test_create_table_integer_default_expr( row = await db.execute_write_fn( lambda conn: _insert_and_fetch_created( - conn, table, f"insert into {escape_sqlite(table)} default values" + conn, table, "insert into {} default values".format(escape_sqlite(table)) ) ) assert row[0] > minimum_value @@ -2412,12 +2331,13 @@ async def test_create_table_column_validation(ds_write, column, expected_error): ) if expected_error: assert response.status_code == 400 - assert response.json() == error_body([expected_error], 400) + assert response.json() == {"ok": False, "errors": [expected_error]} else: assert response.status_code == 400 - assert response.json() == error_body( - ["Could not detect single primary key for table 'owners'"], 400 - ) + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'owners'"], + } @pytest.mark.asyncio @@ -2455,9 +2375,10 @@ async def test_create_table_foreign_key_without_fk_column_requires_single_pk(ds_ headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == error_body( - ["Could not detect single primary key for table 'accounts'"], 400 - ) + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'accounts'"], + } @pytest.mark.asyncio @@ -2607,9 +2528,10 @@ async def test_create_table_error_if_pk_changed(ds_write): headers=_headers(token), ) assert second_response.status_code == 400 - assert second_response.json() == error_body( - ["pk cannot be changed for existing table"], 400 - ) + assert second_response.json() == { + "ok": False, + "errors": ["pk cannot be changed for existing table"], + } @pytest.mark.asyncio @@ -2633,9 +2555,10 @@ async def test_create_table_error_rows_twice_with_duplicates(ds_write): headers=_headers(token), ) assert second_response.status_code == 400 - assert second_response.json() == error_body( - ["UNIQUE constraint failed: test_create_twice.id"], 400 - ) + assert second_response.json() == { + "ok": False, + "errors": ["UNIQUE constraint failed: test_create_twice.id"], + } @pytest.mark.asyncio @@ -2658,8 +2581,6 @@ async def test_method_not_allowed(ds_write, path): assert response.json() == { "ok": False, "error": "Method not allowed", - "errors": ["Method not allowed"], - "status": 405, } @@ -2727,9 +2648,10 @@ async def test_create_using_alter_against_existing_table( ) if not has_alter_permission: assert response2.status_code == 403 - assert response2.json() == error_body( - ["Permission denied: need alter-table"], 403 - ) + assert response2.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } else: assert response2.status_code == 201 diff --git a/tests/test_auth.py b/tests/test_auth.py index e7a5402e..5868a21c 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,17 +1,14 @@ -import time - -import pytest from bs4 import BeautifulSoup as Soup +from .utils import cookie_was_deleted, last_event from click.testing import CliRunner - +from datasette.utils import baseconv from datasette.cli import cli from datasette.resources import ( DatabaseResource, TableResource, ) -from datasette.utils import baseconv - -from .utils import cookie_was_deleted, last_event +import pytest +import time @pytest.mark.asyncio @@ -207,7 +204,7 @@ def test_auth_create_token( assert response2.status == 200 if errors: for error in errors: - assert f'

{error}

' in response2.text + assert '

{}

'.format(error) in response2.text else: # Check create-token event event = last_event(app_client.ds) @@ -231,7 +228,7 @@ def test_auth_create_token( # And test that token response3 = app_client.get( "/-/actor.json", - headers={"Authorization": "Bearer {}".format(f"dstok_{token}")}, + headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, ) assert response3.status == 200 assert response3.json["actor"]["id"] == "test" @@ -239,12 +236,10 @@ def test_auth_create_token( @pytest.mark.asyncio async def test_auth_create_token_not_allowed_for_tokens(ds_client): - ds_tok = ds_client.ds.sign( - {"a": "test", "token": "dstok", "t": int(time.time())}, "token" - ) + ds_tok = ds_client.ds.sign({"a": "test", "token": "dstok"}, "token") response = await ds_client.get( "/-/create-token", - headers={"Authorization": f"Bearer dstok_{ds_tok}"}, + headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, ) assert response.status_code == 403 @@ -289,17 +284,17 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): elif scenario == "invalid_token": token = "invalid" if token: - token = f"dstok_{token}" + token = "dstok_{}".format(token) if scenario == "allow_signed_tokens_off": ds_client.ds._settings["allow_signed_tokens"] = False headers = {} if token: - headers["Authorization"] = f"Bearer {token}" + headers["Authorization"] = "Bearer {}".format(token) response = await ds_client.get("/-/actor.json", headers=headers) try: if should_work: data = response.json() - assert data.keys() == {"ok", "actor"} + assert data.keys() == {"actor"} actor = data["actor"] expected_keys = {"id", "token"} if scenario != "valid_unlimited_token": @@ -309,16 +304,8 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): assert actor["token"] == "dstok" if scenario != "valid_unlimited_token": assert isinstance(actor["token_expires"], int) - elif scenario == "no_token": - # No credentials presented - request proceeds as anonymous - assert response.json() == {"ok": True, "actor": None} else: - # Invalid credentials presented - hard 401 - assert response.status_code == 401 - data = response.json() - assert data["ok"] is False - assert data["status"] == 401 - assert response.headers["www-authenticate"].startswith("Bearer") + assert response.json() == {"actor": None} finally: ds_client.ds._settings["allow_signed_tokens"] = True @@ -341,7 +328,7 @@ def test_cli_create_token(app_client, expires): assert details.keys() == expected_keys assert details["a"] == "test" response = app_client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) if expires is None or expires > 0: expected_actor = { @@ -350,11 +337,10 @@ def test_cli_create_token(app_client, expires): } if expires and expires > 0: expected_actor["token_expires"] = details["t"] + expires - assert response.json == {"ok": True, "actor": expected_actor} + assert response.json == {"actor": expected_actor} else: - # Expired token - hard 401 - assert response.status == 401 - assert response.json["ok"] is False + expected_actor = None + assert response.json == {"actor": expected_actor} @pytest.mark.asyncio diff --git a/tests/test_autocomplete.py b/tests/test_autocomplete.py index 194fcf01..76b9c902 100644 --- a/tests/test_autocomplete.py +++ b/tests/test_autocomplete.py @@ -25,14 +25,13 @@ async def test_autocomplete_single_pk_exact_match_and_label_order(): assert response.status_code == 200 assert response.json() == { - "ok": True, "rows": [ {"pks": {"id": 2}, "label": "Longer non-label pk match"}, {"pks": {"id": 20}, "label": "2"}, {"pks": {"id": 21}, "label": "22"}, {"pks": {"id": 3}, "label": "A label containing 2"}, {"pks": {"id": 200}, "label": "A"}, - ], + ] } @@ -53,12 +52,12 @@ async def test_autocomplete_blank_q_returns_no_results(): response = await ds.client.get("/autocomplete_blank/people/-/autocomplete?q=") assert response.status_code == 200 - assert response.json() == {"ok": True, "rows": []} + assert response.json() == {"rows": []} response = await ds.client.get("/autocomplete_blank/people/-/autocomplete") assert response.status_code == 200 - assert response.json() == {"ok": True, "rows": []} + assert response.json() == {"rows": []} @pytest.mark.asyncio @@ -82,12 +81,11 @@ async def test_autocomplete_initial_returns_latest_rows(): assert response.status_code == 200 assert response.json() == { - "ok": True, "rows": [ {"pks": {"id": 3}, "label": "Cleo"}, {"pks": {"id": 2}, "label": "Bob"}, {"pks": {"id": 1}, "label": "Alice"}, - ], + ] } response = await ds.client.get( @@ -96,12 +94,11 @@ async def test_autocomplete_initial_returns_latest_rows(): assert response.status_code == 200 assert response.json() == { - "ok": True, "rows": [ {"pks": {"id": 3}, "label": "Cleo"}, {"pks": {"id": 2}, "label": "Bob"}, {"pks": {"id": 1}, "label": "Alice"}, - ], + ] } @@ -124,10 +121,9 @@ async def test_autocomplete_escapes_like_characters(): assert response.status_code == 200 assert response.json() == { - "ok": True, "rows": [ {"pks": {"id": 1}, "label": "100% real"}, - ], + ] } @@ -153,12 +149,11 @@ async def test_autocomplete_compound_pk_searches_all_pk_columns(): assert response.status_code == 200 assert response.json() == { - "ok": True, "rows": [ {"pks": {"country": "mx", "code": "ca"}, "label": "Campeche"}, {"pks": {"country": "us", "code": "ca"}, "label": "California"}, {"pks": {"country": "ca", "code": "bc"}, "label": "British Columbia"}, - ], + ] } @@ -189,10 +184,9 @@ async def test_autocomplete_primary_key_called_label(): assert response.status_code == 200 assert response.json() == { - "ok": True, "rows": [ {"pks": {"label": "abc"}, "label": "Display value"}, - ], + ] } @@ -252,9 +246,8 @@ async def test_autocomplete_timeout_uses_prefix_fallback(monkeypatch): assert timeout_was_simulated data = response.json() assert data == { - "ok": True, "rows": [ {"pks": {"id": f"item-1999{i:02d}"}, "label": f"name 1999{i:02d}"} for i in range(10) - ], + ] } diff --git a/tests/test_base_view.py b/tests/test_base_view.py index b46f7ce1..2cd4d601 100644 --- a/tests/test_base_view.py +++ b/tests/test_base_view.py @@ -1,10 +1,8 @@ -import json - -import pytest - +from datasette.views.base import View from datasette import Request, Response from datasette.app import Datasette -from datasette.views.base import View +import json +import pytest class GetView(View): @@ -55,8 +53,6 @@ async def test_get_view(): assert json.loads(post_json_response.body) == { "ok": False, "error": "Method not allowed", - "errors": ["Method not allowed"], - "status": 405, } assert post_json_response.status == 405 diff --git a/tests/test_cli.py b/tests/test_cli.py index fbd4a8a9..f86d6909 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,28 +1,23 @@ +from .fixtures import ( + make_app_client, + TestClient as _TestClient, + EXPECTED_PLUGINS, +) +from datasette.app import SETTINGS +from datasette.plugins import DEFAULT_PLUGINS, pm +from datasette.cli import cli, serve +from datasette.version import __version__ +from datasette.utils import tilde_encode +from datasette.utils.sqlite import sqlite3 +from click.testing import CliRunner import io import json import pathlib +import pytest import sys import textwrap from unittest import mock -import pytest -from click.testing import CliRunner - -from datasette.app import SETTINGS -from datasette.cli import cli, serve -from datasette.plugins import DEFAULT_PLUGINS, pm -from datasette.utils import tilde_encode -from datasette.utils.sqlite import sqlite3 -from datasette.version import __version__ - -from .fixtures import ( - EXPECTED_PLUGINS, - make_app_client, -) -from .fixtures import ( - TestClient as _TestClient, -) - def test_inspect_cli(app_client): runner = CliRunner() @@ -390,9 +385,7 @@ def test_setting_boolean_validation_false_values(value): ) # Should be forbidden (setting is false) assert result.exit_code == 1, result.output - error = json.loads(result.output) - assert error["ok"] is False - assert error["status"] == 403 + assert "Forbidden" in result.output @pytest.mark.parametrize("value", ("on", "true", "1")) @@ -432,9 +425,8 @@ def test_setting_default_allow_sql(default_allow_sql): assert json.loads(result.output)["rows"][0] == {"21": 21} else: assert result.exit_code == 1, result.output - error = json.loads(result.output) - assert error["ok"] is False - assert error["status"] == 403 + # This isn't JSON at the moment, maybe it should be though + assert "Forbidden" in result.output def test_sql_errors_logged_to_stderr(): @@ -452,7 +444,7 @@ def test_serve_create(tmpdir): cli, [str(db_path), "--create", "--get", "/-/databases.json"] ) assert result.exit_code == 0, result.output - databases = json.loads(result.output)["databases"] + databases = json.loads(result.output) assert { "name": "does_not_exist_yet", "is_mutable": True, @@ -465,7 +457,7 @@ def test_serve_create(tmpdir): @pytest.mark.parametrize("argument", ("-c", "--config")) @pytest.mark.parametrize("format_", ("json", "yaml")) def test_serve_config(tmpdir, argument, format_): - config_path = tmpdir / f"datasette.{format_}" + config_path = tmpdir / "datasette.{}".format(format_) config_path.write_text( ( "settings:\n default_page_size: 5\n" @@ -501,7 +493,7 @@ def test_serve_duplicate_database_names(tmpdir): conn.close() result = runner.invoke(cli, [db_1_path, db_2_path, "--get", "/-/databases.json"]) assert result.exit_code == 0, result.output - databases = json.loads(result.output)["databases"] + databases = json.loads(result.output) assert {db["name"] for db in databases} == {"db", "db_2"} @@ -518,13 +510,13 @@ def test_weird_database_names(tmpdir, filename): result1 = runner.invoke(cli, [db_path, "--get", "/"]) assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] - expected_link = ( - f'{filename_no_stem}' + expected_link = '{}'.format( + tilde_encode(filename_no_stem), filename_no_stem ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", f"/{tilde_encode(filename_no_stem)}"] + cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))] ) assert result2.exit_code == 0, result2.output @@ -593,7 +585,7 @@ def test_duplicate_database_files_error(tmpdir): cli, ["serve", other_db_path, str(config_dir), "--get", "/-/databases.json"] ) assert result4.exit_code == 0 - databases = json.loads(result4.output)["databases"] + databases = json.loads(result4.output) assert {db["name"] for db in databases} == {"other", "data"} # Test that multiple directories raise an error diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 01b84f59..dc852201 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,10 +1,8 @@ -import json -import textwrap - -from click.testing import CliRunner - from datasette.cli import cli from datasette.plugins import pm +from click.testing import CliRunner +import textwrap +import json def test_serve_with_get(tmp_path_factory): @@ -46,9 +44,9 @@ def test_serve_with_get(tmp_path_factory): # Annoyingly that new test plugin stays resident - we need # to manually unregister it to avoid conflict with other tests - to_unregister = next( + to_unregister = [ p for p in pm.get_plugins() if p.__name__ == "init_for_serve_with_get.py" - ) + ][0] pm.unregister(to_unregister) @@ -97,10 +95,7 @@ def test_serve_with_get_and_token(): ], ) assert 0 == result2.exit_code, result2.output - assert json.loads(result2.output) == { - "ok": True, - "actor": {"id": "root", "token": "dstok"}, - } + assert json.loads(result2.output) == {"actor": {"id": "root", "token": "dstok"}} def test_serve_with_get_exit_code_for_error(): @@ -135,9 +130,8 @@ def test_serve_get_actor(): ) assert result.exit_code == 0 assert json.loads(result.output) == { - "ok": True, "actor": { "id": "root", "extra": "x", - }, + } } diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index b76180fd..47f23c08 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,8 +1,6 @@ -import socket -import time - import httpx import pytest +import socket @pytest.mark.serial @@ -29,119 +27,3 @@ def test_serve_unix_domain_socket(ds_unix_domain_socket_server): "path": "/_memory", "tables": [], }.items() <= response.json().items() - - -# Shaped after datasette-litestream's startup hook, which schedules a -# background task with asyncio.get_running_loop().create_task(...): -# https://github.com/datasette/datasette-litestream -MARKER_TASK_PLUGIN = """ -import asyncio -from datasette import hookimpl -from datasette.utils.asgi import Response - - -@hookimpl -def startup(datasette): - datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1 - - async def _mark(): - # Must await before setting the flag: a task with no internal - # await point could finish on the throwaway loop before it - # closed, masking the regression this test guards against. - await asyncio.sleep(0.2) - datasette._marker_task_ran = True - - asyncio.get_running_loop().create_task(_mark()) - - -@hookimpl -def register_routes(): - async def marker_status(datasette): - return Response.json( - { - "marker_task_ran": getattr(datasette, "_marker_task_ran", False), - "startup_calls": getattr(datasette, "_startup_calls", 0), - } - ) - - return [(r"^/-/marker-task-ran$", marker_status)] -""" - - -STARTUP_ERROR_PLUGIN = """ -from datasette import hookimpl -from datasette.utils import StartupError - - -@hookimpl -def startup(datasette): - raise StartupError("boom from plugin") -""" - - -@pytest.mark.serial -def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins): - """ - Litestream-shaped regression test: a startup hook that does - asyncio.get_running_loop().create_task(...) must have that task - actually execute before/while the server is handling requests. This - only holds if invoke_startup() and uvicorn.Server.serve() share one - event loop. This test fails against unmodified main, where - invoke_startup() runs on a throwaway loop that is closed before - uvicorn opens its own loop to serve. - """ - _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN}) - # The fixture has already waited for the server to answer requests. The - # marker task deliberately awaits before setting its flag, so poll for a - # moment rather than assuming it landed before the first request arrived. - deadline = time.time() + 3.0 - payload = {} - while time.time() < deadline: - payload = httpx.get( - f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0 - ).json() - if payload["marker_task_ran"]: - break - time.sleep(0.05) - assert payload.get("marker_task_ran"), ( - "The startup hook's asyncio.create_task(...) never ran - " - "invoke_startup() and the server are not sharing an event loop" - ) - # Polling above means this test would also pass if the startup hook were - # re-run on the serving loop by the first-request fallback - which would - # hide exactly the bug being tested. invoke_startup() is idempotent today - # so that cannot happen; assert it explicitly so that if the idempotency - # guard is ever removed this test fails loudly instead of silently - # becoming a no-op. - assert payload["startup_calls"] == 1, ( - "startup hook ran {} times - the marker may have been set by a " - "re-run on the serving loop rather than by the original task".format( - payload["startup_calls"] - ) - ) - - -@pytest.mark.serial -def test_startup_error_fails_fast_before_port_binds(serve_with_plugins): - """ - A "startup" plugin hook that raises StartupError must fail fast: print - the message, exit non-zero, and never accept a connection on the port - - the failure must happen before uvicorn.Server binds the socket. - """ - proc, port = serve_with_plugins( - {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False - ) - stdout, _ = proc.communicate(timeout=15) - output = stdout.decode("utf-8") - assert proc.returncode not in (0, None), output - assert "boom from plugin" in output, output - - # Nothing is listening on the port now the process has exited. This - # confirms the socket was not left bound; on its own it cannot prove the - # failure preceded the bind, since a port nothing ever touched also - # refuses connections. - with ( - pytest.raises(OSError), - socket.create_connection(("127.0.0.1", port), timeout=0.2), - ): - pass diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 50c6daed..45a9e7d1 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -1,11 +1,7 @@ import json import logging -import time -import markupsafe -import pytest from bs4 import BeautifulSoup as Soup - from datasette.app import Datasette from datasette.column_types import ( ColumnType, @@ -13,7 +9,11 @@ from datasette.column_types import ( ) from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.utils import StartupError, error_body, sqlite3 +from datasette.utils import sqlite3 +from datasette.utils import StartupError +import markupsafe +import pytest +import time @pytest.fixture @@ -104,7 +104,7 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", } @@ -322,6 +322,12 @@ async def test_clear_column_type_api(ds_ct): "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ], ), + ( + {"column": "title", "column_type": {"type": "email"}}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), ( [], None, @@ -407,7 +413,11 @@ async def test_set_column_type_api_errors( kwargs = { "headers": { "Authorization": f"Bearer {token}", - "Content-Type": "application/json", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), } } if special_case == "invalid_json": @@ -416,7 +426,7 @@ async def test_set_column_type_api_errors( kwargs["json"] = body response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs) assert response.status_code == expected_status - assert response.json() == error_body(expected_errors, expected_status) + assert response.json() == {"ok": False, "errors": expected_errors} @pytest.mark.asyncio diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 00540464..0a9b30d8 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,12 +1,10 @@ import json import pathlib - import pytest from datasette.app import Datasette -from datasette.utils import StartupError from datasette.utils.sqlite import sqlite3 - +from datasette.utils import StartupError from .fixtures import TestClient as _TestClient PLUGIN = """ @@ -111,10 +109,9 @@ def test_settings(config_dir_client): def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status - plugins = response.json - assert "hooray.py" in {p["name"] for p in plugins} - assert "non_py_file.txt" not in {p["name"] for p in plugins} - assert "mypy_cache" not in {p["name"] for p in plugins} + assert "hooray.py" in {p["name"] for p in response.json} + assert "non_py_file.txt" not in {p["name"] for p in response.json} + assert "mypy_cache" not in {p["name"] for p in response.json} def test_templates_and_plugin(config_dir_client): @@ -139,7 +136,7 @@ def test_static_directory_browsing_not_allowed(config_dir_client): def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status - databases = response.json["databases"] + databases = response.json assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index ffd0870c..11e53224 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -1,9 +1,7 @@ -import sqlite3 -import urllib - -from click.testing import CliRunner - from datasette.cli import cli +from click.testing import CliRunner +import urllib +import sqlite3 def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled): @@ -42,7 +40,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory): db_dir = tmp_path_factory.mktemp("dbs") dbs = [] for i in range(11): - path = str(db_dir / f"db_{i}.db") + path = str(db_dir / "db_{}.db".format(i)) conn = sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 6c78f69d..2fcfb216 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -44,7 +44,7 @@ async def _run_middleware(scope): await mw(scope, None, send) if inner_called: return ("allowed",) - start = next(m for m in sent if m["type"] == "http.response.start") + start = [m for m in sent if m["type"] == "http.response.start"][0] return ("blocked", start["status"]) diff --git a/tests/test_csv.py b/tests/test_csv.py index 7758a3c0..a2f03776 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,9 +1,7 @@ -import urllib.parse - -import pytest -from bs4 import BeautifulSoup as Soup - from datasette.app import Datasette +from bs4 import BeautifulSoup as Soup +import pytest +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 32cfc43d..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -1,7 +1,5 @@ import pathlib - import pytest - from .fixtures import make_app_client TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates") diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index f456a17f..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -1,5 +1,4 @@ import pytest - from datasette.app import Datasette from datasette.resources import DatabaseResource, TableResource diff --git a/tests/test_docs.py b/tests/test_docs.py index 16df6a46..13b3a549 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -2,22 +2,20 @@ Tests to ensure certain things are documented. """ -import re -from pathlib import Path - -import pytest - -import datasette.fixtures # noqa: F401 from datasette import app, utils +import datasette.fixtures # noqa: F401 from datasette.app import Datasette from datasette.filters import Filters +from pathlib import Path +import pytest +import re docs_path = Path(__file__).parent.parent / "docs" label_re = re.compile(r"\.\. _([^\s:]+):") def get_headings(content, underline="-"): - heading_re = re.compile(rf"(\w+)(\([^)]*\))?\n\{underline}+\n") + heading_re = re.compile(r"(\w+)(\([^)]*\))?\n\{}+\n".format(underline)) return {h[0] for h in heading_re.findall(content)} @@ -250,7 +248,7 @@ async def test_homepage(): async def test_actor_is_null(): ds = Datasette(memory=True) response = await ds.client.get("/-/actor.json") - assert response.json() == {"ok": True, "actor": None} + assert response.json() == {"actor": None} # -- end test_actor_is_null -- @@ -260,5 +258,5 @@ async def test_signed_cookie_actor(): ds = Datasette(memory=True) cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} response = await ds.client.get("/-/actor.json", cookies=cookies) - assert response.json() == {"ok": True, "actor": {"id": "root"}} + assert response.json() == {"actor": {"id": "root"}} # -- end test_signed_cookie_actor -- diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index 4a0014b4..613160ac 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -1,10 +1,9 @@ # fmt: off # -- start datasette_with_plugin_fixture -- -import pytest -import pytest_asyncio - from datasette import hookimpl from datasette.app import Datasette +import pytest +import pytest_asyncio @pytest_asyncio.fixture diff --git a/tests/test_error_shape.py b/tests/test_error_shape.py deleted file mode 100644 index 94c9a7c9..00000000 --- a/tests/test_error_shape.py +++ /dev/null @@ -1,751 +0,0 @@ -""" -Tests for the canonical JSON error shape. - -Every JSON error response from Datasette should use one shape: - - { - "ok": false, - "error": "", - "errors": ["", ...], - "status": - } - -Additional context keys (for example "rows" and "truncated" on SQL errors) -are permitted, but "ok", "error", "errors" and "status" must always be -present and the legacy "title" key must not be. - -https://github.com/simonw/datasette/issues - 1.0 API consistency -""" - -import time - -import pytest - -from datasette.app import Datasette -from datasette.utils import sqlite3 - - -def assert_canonical_error(response, expected_status): - assert response.status_code == expected_status - data = response.json() - assert data["ok"] is False - assert isinstance(data["error"], str) - assert data["error"] - assert isinstance(data["errors"], list) - assert data["errors"] - assert all(isinstance(message, str) for message in data["errors"]) - assert data["error"] == "; ".join(data["errors"]) - assert data["status"] == expected_status - assert "title" not in data - return data - - -@pytest.fixture -def ds_error_shape(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = str(db_directory / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("vacuum") - conn.execute("create table docs (id integer primary key, title text)") - conn.close() - ds = Datasette([db_path]) - ds.root_enabled = True - yield ds - ds.close() - - -# Shape 1: the exception handler (handle_exception.py) - - -@pytest.mark.asyncio -async def test_not_found_error_shape(ds_client): - response = await ds_client.get("/fixtures/no_such_table.json") - assert_canonical_error(response, 404) - - -@pytest.mark.asyncio -async def test_datasette_error_with_title_omits_title_key(ds_client): - # DatasetteError(title="Invalid SQL") previously leaked a "title" key - response = await ds_client.get( - "/fixtures/-/query.json?sql=update+facetable+set+state+=+1" - ) - data = assert_canonical_error(response, 400) - assert data["errors"] == ["Statement must be a SELECT"] - - -# Shape 2: the _error() helper (views/base.py) - write API and friends - - -@pytest.mark.asyncio -async def test_write_api_validation_error_shape(ds_error_shape): - token = "dstok_{}".format( - ds_error_shape.sign( - {"a": "root", "token": "dstok", "t": 0}, - namespace="token", - ) - ) - response = await ds_error_shape.client.post( - "/data/docs/-/insert", - json={"rows": [{"nope": 1}, {"also_nope": 2}]}, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - ) - data = assert_canonical_error(response, 400) - # Multiple messages: errors keeps them all, error joins them - assert len(data["errors"]) == 2 - assert data["errors"][0].startswith("Row 0") - assert data["errors"][1].startswith("Row 1") - - -@pytest.mark.asyncio -async def test_write_api_permission_denied_shape(ds_error_shape): - response = await ds_error_shape.client.post( - "/data/docs/-/insert", - json={"rows": [{"title": "hello"}]}, - headers={"Content-Type": "application/json"}, - ) - assert_canonical_error(response, 403) - - -# Shape 3: the JSON renderer (renderer.py) - - -@pytest.mark.asyncio -async def test_sql_error_shape_keeps_context_keys(ds_client): - response = await ds_client.get( - "/fixtures/-/query.json?sql=select+*+from+no_such_table" - ) - data = assert_canonical_error(response, 400) - # Renderer errors keep their context keys - assert data["rows"] == [] - assert "truncated" in data - - -@pytest.mark.asyncio -async def test_invalid_shape_error_shape(ds_client): - response = await ds_client.get("/fixtures/-/query.json?sql=select+1&_shape=bananas") - data = assert_canonical_error(response, 400) - assert data["errors"] == ["Invalid _shape: bananas"] - - -@pytest.mark.asyncio -async def test_shape_object_on_query_is_a_400_error(ds_client): - # Previously returned HTTP 200 with an ok: false body - response = await ds_client.get("/fixtures/-/query.json?sql=select+1&_shape=object") - data = assert_canonical_error(response, 400) - assert data["errors"] == ["_shape=object is only available on tables"] - - -# Shape 4: bare {"error": ...} from the permission debug endpoints - - -@pytest.mark.asyncio -async def test_allowed_missing_action_error_shape(ds_client): - response = await ds_client.get("/-/allowed.json") - data = assert_canonical_error(response, 400) - assert data["errors"] == ["action parameter is required"] - - -@pytest.mark.asyncio -async def test_allowed_unknown_action_error_shape(ds_client): - response = await ds_client.get("/-/allowed.json?action=no_such_action") - assert_canonical_error(response, 404) - - -@pytest.mark.asyncio -async def test_check_unknown_action_error_shape(ds_error_shape): - response = await ds_error_shape.client.get( - "/-/check.json?action=no_such_action", - actor={"id": "root"}, - ) - assert_canonical_error(response, 404) - - -@pytest.mark.asyncio -async def test_rules_missing_action_error_shape(ds_error_shape): - response = await ds_error_shape.client.get( - "/-/rules.json", - actor={"id": "root"}, - ) - data = assert_canonical_error(response, 400) - assert data["errors"] == ["action parameter is required"] - - -# Other stragglers - - -@pytest.mark.asyncio -async def test_method_not_allowed_error_shape(ds_client): - response = await ds_client.post("/fixtures.json") - assert_canonical_error(response, 405) - - -@pytest.mark.asyncio -async def test_schema_unknown_database_error_shape(ds_client): - response = await ds_client.get("/no_such_db/-/schema.json") - assert_canonical_error(response, 404) - - -# Forbidden responses (the default forbidden() hook) - - -@pytest.fixture -def ds_forbidden(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = str(db_directory / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("vacuum") - conn.execute("create table docs (id integer primary key, title text)") - conn.close() - ds = Datasette( - [db_path], - config={"databases": {"data": {"tables": {"docs": {"allow": {"id": "root"}}}}}}, - ) - ds.root_enabled = True - yield ds - ds.close() - - -@pytest.mark.asyncio -async def test_forbidden_json_path_returns_canonical_json(ds_forbidden): - response = await ds_forbidden.client.get("/data/docs.json") - data = assert_canonical_error(response, 403) - assert "permission" in data["error"].lower() - - -@pytest.mark.asyncio -async def test_forbidden_accept_json_returns_canonical_json(ds_forbidden): - response = await ds_forbidden.client.get( - "/data/docs", headers={"Accept": "application/json"} - ) - assert_canonical_error(response, 403) - - -@pytest.mark.asyncio -async def test_forbidden_html_path_still_returns_html(ds_forbidden): - response = await ds_forbidden.client.get("/data/docs") - assert response.status_code == 403 - assert response.headers["content-type"].startswith("text/html") - - -@pytest.mark.asyncio -async def test_forbidden_json_path_allowed_actor_still_works(ds_forbidden): - response = await ds_forbidden.client.get("/data/docs.json", actor={"id": "root"}) - assert response.status_code == 200 - assert response.json()["ok"] is True - - -# Write canned queries: SQL failures must not return HTTP 200 - - -@pytest.fixture -def ds_write_query(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = str(db_directory / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("vacuum") - conn.execute("create table docs (id integer primary key, title text)") - conn.close() - ds = Datasette( - [db_path], - config={ - "databases": { - "data": { - "queries": { - "add_doc": { - "sql": ( - "insert into docs (id, title)" " values (:id, :title)" - ), - "write": True, - }, - "add_doc_custom_error": { - "sql": ( - "insert into docs (id, title)" " values (:id, :title)" - ), - "write": True, - "on_error_message": "Custom error message", - "on_error_redirect": "/data", - }, - } - } - } - }, - ) - yield ds - ds.close() - - -@pytest.mark.asyncio -async def test_write_query_success_returns_200(ds_write_query): - response = await ds_write_query.client.post( - "/data/add_doc", - json={"id": 1, "title": "One"}, - headers={"Accept": "application/json"}, - ) - assert response.status_code == 200 - data = response.json() - assert data["ok"] is True - assert data["message"] == "Query executed, 1 row affected" - assert data["redirect"] is None - - -@pytest.mark.asyncio -async def test_write_query_sql_failure_returns_400(ds_write_query): - for _ in range(2): - response = await ds_write_query.client.post( - "/data/add_doc", - json={"id": 1, "title": "One"}, - headers={"Accept": "application/json"}, - ) - data = assert_canonical_error(response, 400) - assert "UNIQUE constraint failed" in data["error"] - # The redirect context key from the canned query flow is preserved - assert data["redirect"] is None - - -@pytest.mark.asyncio -async def test_write_query_failure_uses_on_error_message_and_redirect( - ds_write_query, -): - for _ in range(2): - response = await ds_write_query.client.post( - "/data/add_doc_custom_error", - json={"id": 1, "title": "One"}, - headers={"Accept": "application/json"}, - ) - data = assert_canonical_error(response, 400) - assert data["error"] == "Custom error message" - assert data["redirect"] == "/data" - - -@pytest.mark.asyncio -async def test_write_query_forbidden_is_canonical_403(ds_write_query): - # An untrusted write query run by an actor without execute-write-sql - # raises Forbidden, handled by the forbidden() hook - await ds_write_query.invoke_startup() - await ds_write_query.add_query( - "data", - name="untrusted_add", - sql="insert into docs (id, title) values (:id, :title)", - is_write=True, - is_trusted=False, - source="user", - owner_id="someone", - ) - response = await ds_write_query.client.post( - "/data/untrusted_add", - json={"id": 5, "title": "Five"}, - headers={"Accept": "application/json"}, - actor={"id": "someone"}, - ) - assert_canonical_error(response, 403) - - -@pytest.mark.asyncio -async def test_write_query_rejected_operation_is_canonical_403(ds_write_query): - # A rejected operation (VACUUM) raises QueryWriteRejected, handled by - # the dedicated branch in QueryView.post - root has execute-write-sql - ds_write_query.root_enabled = True - await ds_write_query.invoke_startup() - await ds_write_query.add_query( - "data", - name="vacuum_it", - sql="vacuum", - is_write=True, - is_trusted=False, - source="user", - owner_id="root", - ) - response = await ds_write_query.client.post( - "/data/vacuum_it", - json={}, - headers={"Accept": "application/json"}, - actor={"id": "root"}, - ) - data = assert_canonical_error(response, 403) - assert data["redirect"] is None - - -# Row delete write failures must be 400, matching row update - - -@pytest.mark.asyncio -async def test_row_delete_write_failure_is_400(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = str(db_directory / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("vacuum") - conn.execute("create table docs (id integer primary key, title text)") - conn.execute("insert into docs (id, title) values (1, 'One')") - conn.execute( - "create trigger no_delete before delete on docs " - "begin select raise(abort, 'deletes are blocked'); end" - ) - conn.commit() - conn.close() - ds = Datasette([db_path]) - ds.root_enabled = True - try: - response = await ds.client.post( - "/data/docs/1/-/delete", - json={}, - headers={"Content-Type": "application/json"}, - actor={"id": "root"}, - ) - data = assert_canonical_error(response, 400) - assert "deletes are blocked" in data["error"] - finally: - ds.close() - - -# Invalid bearer tokens must produce 401, not silent anonymous access - - -@pytest.mark.asyncio -async def test_expired_token_returns_401(ds_error_shape): - token = "dstok_{}".format( - ds_error_shape.sign( - {"a": "root", "t": int(time.time()) - 2000, "d": 1000}, - namespace="token", - ) - ) - response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} - ) - data = assert_canonical_error(response, 401) - assert "expired" in data["error"].lower() - assert response.headers["www-authenticate"].startswith("Bearer") - - -@pytest.mark.asyncio -async def test_bad_signature_token_returns_401(ds_error_shape): - response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": "Bearer dstok_garbage"} - ) - assert_canonical_error(response, 401) - assert response.headers["www-authenticate"].startswith("Bearer") - - -@pytest.mark.asyncio -async def test_unrecognized_token_prefix_stays_anonymous(ds_error_shape): - # No registered handler claims this token - it might belong to a - # plugin's actor_from_request hook, so it must not hard-fail - response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": "Bearer sometoken_abc"} - ) - assert response.status_code == 200 - assert response.json() == {"ok": True, "actor": None} - - -@pytest.mark.asyncio -async def test_valid_token_still_authenticates(ds_error_shape): - token = "dstok_{}".format( - ds_error_shape.sign( - {"a": "root", "t": int(time.time())}, - namespace="token", - ) - ) - response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} - ) - assert response.status_code == 200 - assert response.json()["actor"]["id"] == "root" - - -@pytest.mark.asyncio -async def test_bad_token_beats_valid_cookie(ds_error_shape): - # A malformed Authorization header is a hard error even if a valid - # ds_actor cookie is also present - response = await ds_error_shape.client.get( - "/-/actor.json", - headers={"Authorization": "Bearer dstok_garbage"}, - cookies={"ds_actor": ds_error_shape.client.actor_cookie({"id": "root"})}, - ) - assert_canonical_error(response, 401) - - -@pytest.mark.asyncio -async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = str(db_directory / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("vacuum") - conn.close() - ds = Datasette([db_path], settings={"allow_signed_tokens": False}) - try: - token = "dstok_{}".format( - ds.sign({"a": "root", "t": int(time.time())}, namespace="token") - ) - response = await ds.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} - ) - data = assert_canonical_error(response, 401) - assert "not enabled" in data["error"] - finally: - ds.close() - - -# GET /db/-/query without SQL: 400 for data formats, HTML editor stays 200 - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path", - ( - "/fixtures/-/query.json", - "/fixtures/-/query.json?sql=", - ), -) -async def test_query_json_without_sql_is_400(ds_client, path): - response = await ds_client.get(path) - data = assert_canonical_error(response, 400) - assert data["errors"] == ["?sql= is required"] - - -@pytest.mark.asyncio -async def test_query_html_without_sql_is_still_the_editor(ds_client): - response = await ds_client.get("/fixtures/-/query") - assert response.status_code == 200 - assert response.headers["content-type"].startswith("text/html") - - -# Write API return:true responses use "rows" consistently - - -@pytest.mark.asyncio -async def test_row_update_return_uses_rows_list(ds_error_shape): - await ds_error_shape.client.post( - "/data/docs/-/insert", - json={"row": {"id": 1, "title": "One"}}, - headers={"Content-Type": "application/json"}, - actor={"id": "root"}, - ) - response = await ds_error_shape.client.post( - "/data/docs/1/-/update", - json={"update": {"title": "Updated"}, "return": True}, - headers={"Content-Type": "application/json"}, - actor={"id": "root"}, - ) - assert response.status_code == 200 - data = response.json() - assert data["ok"] is True - assert "row" not in data - assert data["rows"] == [{"id": 1, "title": "Updated"}] - - -# Schema endpoints: no existence oracle, no 500 on unknown database - - -@pytest.mark.asyncio -async def test_schema_endpoints_no_existence_oracle(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = str(db_directory / "data.db") - conn = sqlite3.connect(db_path) - conn.execute("vacuum") - conn.execute("create table docs (id integer primary key)") - conn.close() - ds = Datasette([db_path], default_deny=True) - ds.root_enabled = True - try: - # An actor without view-database cannot distinguish an existing - # database from a missing one - denied_existing = await ds.client.get("/data/-/schema.json") - denied_missing = await ds.client.get("/nope/-/schema.json") - assert denied_existing.status_code == denied_missing.status_code == 403 - - # An authorized actor sees the real thing - root_existing = await ds.client.get("/data/-/schema.json", actor={"id": "root"}) - assert root_existing.status_code == 200 - root_missing = await ds.client.get("/nope/-/schema.json", actor={"id": "root"}) - assert root_missing.status_code == 404 - finally: - ds.close() - - -@pytest.mark.asyncio -async def test_table_schema_unknown_database_is_404_not_500(ds_client): - response = await ds_client.get("/no_such_db/some_table/-/schema.json") - assert_canonical_error(response, 404) - - -# Unknown _extra names are a 400, not silently ignored - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path", - ( - "/fixtures/facetable.json?_extra=nope", - "/fixtures/facetable.json?_extra=count,nope", - "/fixtures/simple_primary_key/1.json?_extra=nope", - "/fixtures/-/query.json?sql=select+1&_extra=nope", - ), -) -async def test_unknown_extra_is_400(ds_client, path): - response = await ds_client.get(path) - data = assert_canonical_error(response, 400) - assert data["errors"] == ["Unknown _extra: nope"] - - -@pytest.mark.asyncio -async def test_html_only_extra_via_json_is_400(ds_client): - # display_rows exists for the HTML view but is not part of the JSON API - response = await ds_client.get("/fixtures/facetable.json?_extra=display_rows") - data = assert_canonical_error(response, 400) - assert data["errors"] == ["Unknown _extra: display_rows"] - - -@pytest.mark.asyncio -async def test_unknown_extra_ignored_on_html_pages(ds_client): - response = await ds_client.get("/fixtures/facetable?_extra=nope") - assert response.status_code == 200 - assert response.headers["content-type"].startswith("text/html") - - -# /-/threads exposes runtime internals and requires permissions-debug - - -@pytest.mark.asyncio -async def test_threads_requires_permissions_debug(ds_error_shape): - denied = await ds_error_shape.client.get("/-/threads.json") - assert_canonical_error(denied, 403) - allowed = await ds_error_shape.client.get("/-/threads.json", actor={"id": "root"}) - assert allowed.status_code == 200 - assert allowed.json()["ok"] is True - - -# _size is the one page-size parameter, with uniform validation - - -@pytest.mark.asyncio -async def test_query_list_size_supports_max_keyword(ds_client): - response = await ds_client.get("/fixtures/-/queries.json?_size=max") - assert response.status_code == 200 - # ds_client runs with max_returned_rows=100 - assert response.json()["limit"] == 100 - - -@pytest.mark.asyncio -async def test_query_list_size_rejects_out_of_range(ds_client): - response = await ds_client.get("/fixtures/-/queries.json?_size=5000") - data = assert_canonical_error(response, 400) - assert data["errors"] == ["_size must be <= 100"] - - -@pytest.mark.asyncio -async def test_query_list_size_rejects_non_integer(ds_client): - response = await ds_client.get("/fixtures/-/queries.json?_size=bananas") - data = assert_canonical_error(response, 400) - assert data["errors"] == ["_size must be a positive integer"] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("endpoint", ("allowed", "rules")) -async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint): - base = f"/-/{endpoint}.json?action=view-instance" - ok = await ds_error_shape.client.get( - base + "&_size=1&_page=1", actor={"id": "root"} - ) - assert ok.status_code == 200 - assert ok.json()["page_size"] == 1 - - max_size = await ds_error_shape.client.get( - base + "&_size=max", actor={"id": "root"} - ) - assert max_size.status_code == 200 - assert max_size.json()["page_size"] == 200 - - too_big = await ds_error_shape.client.get(base + "&_size=500", actor={"id": "root"}) - data = assert_canonical_error(too_big, 400) - assert data["errors"] == ["_size must be <= 200"] - - bad_page = await ds_error_shape.client.get(base + "&_page=0", actor={"id": "root"}) - data = assert_canonical_error(bad_page, 400) - assert data["errors"] == ["_page must be a positive integer"] - - -# Write endpoints parse the body as JSON regardless of Content-Type - - -@pytest.mark.asyncio -async def test_insert_works_without_content_type_header(ds_error_shape): - # Previously a 500 AttributeError - response = await ds_error_shape.client.post( - "/data/docs/-/insert", - content='{"row": {"id": 1, "title": "One"}}', - actor={"id": "root"}, - ) - assert response.status_code == 201 - assert response.json()["rows"][0]["title"] == "One" - - -@pytest.mark.asyncio -async def test_insert_works_with_form_content_type(ds_error_shape): - # Previously 400 "Invalid content-type, must be application/json" - response = await ds_error_shape.client.post( - "/data/docs/-/insert", - content='{"row": {"id": 2, "title": "Two"}}', - headers={"Content-Type": "application/x-www-form-urlencoded"}, - actor={"id": "root"}, - ) - assert response.status_code == 201 - - -@pytest.mark.asyncio -async def test_insert_form_encoded_body_is_invalid_json(ds_error_shape): - response = await ds_error_shape.client.post( - "/data/docs/-/insert", - content="title=Three", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - actor={"id": "root"}, - ) - data = assert_canonical_error(response, 400) - assert data["errors"][0].startswith("Invalid JSON:") - - -@pytest.mark.asyncio -async def test_alter_and_set_column_type_ignore_content_type(ds_error_shape): - alter = await ds_error_shape.client.post( - "/data/docs/-/alter", - content='{"operations": [{"op": "add_column", "args": {"name": "extra"}}]}', - actor={"id": "root"}, - ) - assert alter.status_code == 200, alter.text - sct = await ds_error_shape.client.post( - "/data/docs/-/set-column-type", - content='{"column": "title", "column_type": {"type": "textarea"}}', - actor={"id": "root"}, - ) - assert sct.status_code == 200, sct.text - - -# SQL Interrupted errors carry plain text in JSON, not an HTML fragment - - -@pytest.mark.asyncio -async def test_sql_interrupted_json_error_is_plain_text(ds_client): - response = await ds_client.get( - "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5" - ) - data = assert_canonical_error(response, 400) - assert "<" not in data["error"] - assert data["error"].startswith("SQL query took too long.") - - -@pytest.mark.asyncio -async def test_sql_interrupted_html_page_keeps_rich_error(ds_client): - response = await ds_client.get( - "/fixtures/-/query?sql=select+sleep(0.01)&_timelimit=5" - ) - assert response.status_code == 400 - assert "{expected_show_hide_text})' + expected_show_hide_fragment = '({})'.format( + expected_show_hide_link, expected_show_hide_text ) response = client.get("/_memory/one" + querystring) html = response.text @@ -787,7 +788,10 @@ def test_stored_query_show_hide_metadata_option( )[0] assert show_hide_fragment == expected_show_hide_fragment if expected_hidden: - assert f'' in html + assert ( + ''.format(expected_hidden) + in html + ) else: assert '; rel="alternate"; type="application/json+datasette"' + assert link == '<{}>; rel="alternate"; type="application/json+datasette"'.format( + expected + ) assert ( - f'' + ''.format( + expected + ) in response.text ) @@ -1284,8 +1292,8 @@ async def test_database_color(ds_client): expected_color = ds_client.ds.get_database("fixtures").color # Should be something like #9403e5 expected_fragments = ( - f"10px solid #{expected_color}", - f"border-color: #{expected_color}", + "10px solid #{}".format(expected_color), + "border-color: #{}".format(expected_color), ) assert len(expected_color) == 6 for path in ( @@ -1355,12 +1363,12 @@ async def test_permission_debug_tabs_with_query_string(ds_client): # Test /-/allowed with query string response = await ds_client.get( - "/-/allowed?action=view-table&_size=50", actor=actor + "/-/allowed?action=view-table&page_size=50", actor=actor ) assert response.status_code == 200 # Check that Rules and Check tabs have the query string - assert 'href="/-/rules?action=view-table&_size=50"' in response.text - assert 'href="/-/check?action=view-table&_size=50"' in response.text + assert 'href="/-/rules?action=view-table&page_size=50"' in response.text + assert 'href="/-/check?action=view-table&page_size=50"' in response.text # Playground and Actions should not have query string assert 'href="/-/permissions"' in response.text assert 'href="/-/actions"' in response.text diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index b4bb964d..26d63a92 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,9 +1,5 @@ -import sqlite3 - import pytest - -from datasette.utils import escape_sqlite -from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL +import sqlite_utils # ensure refresh_schemas() gets called before interacting with internal_db @@ -20,53 +16,6 @@ async def test_internal_databases(ds_client): assert databases.rows[0]["database_name"] == "fixtures" -@pytest.mark.asyncio -async def test_internal_migrations_recorded(ds_client): - internal_db = await ensure_internal(ds_client) - migrations = await internal_db.execute(""" - select migration_set, name - from _sqlite_migrations - order by id - """) - assert [tuple(row) for row in migrations.rows] == [ - ("datasette_internal", "0001_initial") - ] - - -@pytest.mark.asyncio -async def test_internal_migrations_adopt_existing_internal_db(tmp_path): - from datasette.app import Datasette - - internal_db_path = str(tmp_path / "internal.db") - conn = sqlite3.connect(internal_db_path) - conn.executescript(INTERNAL_DB_SCHEMA_SQL) - conn.execute( - "insert into metadata_instance (key, value) values (?, ?)", - ("legacy", "preserved"), - ) - conn.commit() - conn.close() - - ds = Datasette(internal=internal_db_path) - await ds.invoke_startup() - internal_db = ds.get_internal_database() - - metadata = await internal_db.execute( - "select key, value from metadata_instance where key = 'legacy'" - ) - assert [tuple(row) for row in metadata.rows] == [("legacy", "preserved")] - migrations = await internal_db.execute(""" - select migration_set, name - from _sqlite_migrations - order by id - """) - assert [tuple(row) for row in migrations.rows] == [ - ("datasette_internal", "0001_initial") - ] - - ds.close() - - @pytest.mark.asyncio async def test_internal_tables(ds_client): internal_db = await ensure_internal(ds_client) @@ -127,62 +76,19 @@ async def test_internal_foreign_key_references(ds_client): internal_db = await ensure_internal(ds_client) def inner(conn): - table_names = [ - row[0] - for row in conn.execute( - "select name from sqlite_master where type = 'table'" - ).fetchall() - ] - - def columns_for_table(table_name): - return { - row[1] - for row in conn.execute( - f"PRAGMA table_info({escape_sqlite(table_name)})" - ).fetchall() - } - - def primary_keys_for_table(table_name): - return [ - name - for _, name in sorted( - (row[5], row[1]) - for row in conn.execute( - f"PRAGMA table_info({escape_sqlite(table_name)})" - ).fetchall() - if row[5] + db = sqlite_utils.Database(conn) + table_names = db.table_names() + for table in db.tables: + for fk in table.foreign_keys: + other_table = fk.other_table + other_column = fk.other_column + message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format( + table.name, fk.column, other_table, other_column ) - ] - - columns_by_table = { - table_name: columns_for_table(table_name) for table_name in table_names - } - - for table_name in table_names: - foreign_key_rows = conn.execute( - f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" - ).fetchall() - foreign_keys_by_id = {} - for foreign_key in foreign_key_rows: - foreign_keys_by_id.setdefault(foreign_key[0], []).append(foreign_key) - - for foreign_key_rows in foreign_keys_by_id.values(): - foreign_key_rows.sort(key=lambda row: row[1]) - other_table = foreign_key_rows[0][2] - other_columns = [row[4] for row in foreign_key_rows] - message = f'Column "{table_name}.{foreign_key_rows[0][3]}" references other table "{other_table}" which does not exist' assert other_table in table_names, message + " (bad table)" - if all(other_column is None for other_column in other_columns): - other_columns = primary_keys_for_table(other_table) - length_message = f'Foreign key from "{table_name}" to "{other_table}" has {len(foreign_key_rows)} columns but references {len(other_columns)} columns' - assert len(other_columns) == len(foreign_key_rows), length_message - - for foreign_key, other_column in zip(foreign_key_rows, other_columns): - column = foreign_key[3] - message = f'Column "{table_name}.{column}" references other column "{other_table}.{other_column}" which does not exist' - assert other_column in columns_by_table[other_table], ( - message + " (bad column)" - ) + assert other_column in db[other_table].columns_dict, ( + message + " (bad column)" + ) await internal_db.execute_fn(inner) @@ -237,10 +143,10 @@ async def test_stale_catalog_entry_database_fix(tmp_path): @pytest.mark.asyncio async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): - import sqlite3 - from datasette.app import Datasette + import sqlite3 + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") bravo_db_path = str(tmp_path / "bravo.db") @@ -285,10 +191,10 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path @pytest.mark.asyncio async def test_orphan_stale_catalog_child_entries_removed(tmp_path): - import sqlite3 - from datasette.app import Datasette + import sqlite3 + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index b1093b1c..bad4e8ca 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -3,23 +3,16 @@ Tests for the datasette.database.Database class """ import asyncio -import uuid from types import SimpleNamespace - -import pytest -import sqlite_utils - from datasette.app import Datasette -from datasette.database import ( - Database, - DatasetteClosedError, - ExecuteWriteResult, - MultipleValues, - Results, - _deliver_write_result, -) -from datasette.utils import Column +from datasette.database import Database, ExecuteWriteResult, Results, MultipleValues +from datasette.database import DatasetteClosedError +from datasette.database import _deliver_write_result from datasette.utils.sqlite import sqlite3, supports_returning +from datasette.utils import Column +import pytest +import time +import uuid requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -50,7 +43,7 @@ async def test_results_first(db): @pytest.mark.parametrize("expected", (True, False)) async def test_results_bool(db, expected): where = "" if expected else "where pk = 0" - results = await db.execute(f"select * from facetable {where}") + results = await db.execute("select * from facetable {}".format(where)) assert bool(results) is expected @@ -622,7 +615,7 @@ async def test_execute_write_block_false(db): "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], ) - await asyncio.sleep(0.1) + time.sleep(0.1) rows = await db.execute("select name from roadside_attractions where pk = 1") assert "Mystery!" == rows.rows[0][0] @@ -640,7 +633,7 @@ async def test_execute_write_with_returning_block_false(db): ) assert isinstance(task_id, uuid.UUID) - await asyncio.sleep(0.1) + time.sleep(0.1) assert ( await db.execute("select name from write_returning_block_false") ).single_value() == "Cleo" @@ -725,51 +718,15 @@ async def test_execute_write_fn_exception(db): await db.execute_write_fn(write_fn) -@pytest.mark.asyncio -@pytest.mark.parametrize("num_sql_threads", (0, 1)) -async def test_execute_write_fn_sqlite_utils_transaction(tmp_path, num_sql_threads): - # A write inside a failing Datasette task must never become visible or - # survive the rollback. Exercise both the synchronous and writer-thread - # paths against a file-backed database so a second connection can observe - # committed state independently. - db_path = tmp_path / "test.db" - sqlite3.connect(db_path).close() - ds = Datasette([str(db_path)], settings={"num_sql_threads": num_sql_threads}) - db = ds.get_database("test") - await db.execute_write("create table items (id integer primary key)") - # This reader is used inside the write callback, which may run on another - # thread, but it is never accessed concurrently. - reader = sqlite3.connect(db_path, check_same_thread=False) - - def insert_then_fail(conn): - # Datasette must open the outer transaction before sqlite-utils writes. - assert conn.in_transaction - sqlite_utils.Database(conn)["items"].insert({"id": 1}) - # If sqlite-utils committed its own transaction, this would return 1. - assert reader.execute("select count(*) from items").fetchone()[0] == 0 - # Simulate a later step failing after the sqlite-utils write succeeded. - raise ValueError("deliberate") - - try: - with pytest.raises(ValueError, match="deliberate"): - await db.execute_write_fn(insert_then_fail) - # The outer transaction must roll back the sqlite-utils write as well. - assert reader.execute("select count(*) from items").fetchone()[0] == 0 - finally: - reader.close() - db.close() - - @pytest.mark.asyncio @pytest.mark.parametrize("param_name", ["conn", "connection", "db", "c"]) async def test_execute_write_fn_accepts_any_single_param_name(db, param_name): # Plugins historically relied on the fact that the callback was invoked # positionally, so any parameter name worked. Preserve that contract. scope = {} - # exec() is how we build a function with a parameterized argument name - exec( # noqa: S102 - f"def write_fn({param_name}):\n" - f" return {param_name}.execute('select 1 + 1').fetchone()[0]", + exec( + "def write_fn({0}):\n" + " return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name), scope, ) write_fn = scope["write_fn"] @@ -793,9 +750,7 @@ async def test_execute_write_fn_with_track_event(db): @pytest.mark.asyncio -# func_only so the budget covers the write-thread call under test, not the -# one-off app_client fixture setup this test may be first to trigger -@pytest.mark.timeout(1, func_only=True) +@pytest.mark.timeout(1) async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") conn = sqlite3.connect(path) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index ed2aeaf0..2eaee3f9 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -9,15 +9,13 @@ import importlib import os import sqlite3 import time - -import pytest -from itsdangerous import BadSignature - from datasette import Context -from datasette.app import Database, Datasette, ResourcesSQL +from datasette.app import Datasette, Database, ResourcesSQL from datasette.database import DatasetteClosedError from datasette.resources import DatabaseResource from datasette.utils import PrefixedUrlString +from itsdangerous import BadSignature +import pytest @pytest.fixture @@ -79,7 +77,9 @@ async def test_static_template_function_hashes_core_asset(tmp_path, monkeypatch) template = ds.get_jinja_environment().from_string("{{ static('demo.js') }}") expected_hash = hashlib.sha256(b"const demo = true;").hexdigest()[:12] - assert await template.render_async() == f"/-/static/demo.js?_hash={expected_hash}" + assert await template.render_async() == "/-/static/demo.js?_hash={}".format( + expected_hash + ) assert isinstance(ds.static("demo.js"), PrefixedUrlString) @@ -101,7 +101,7 @@ def test_static_hash_recalculated_when_cache_headers_disabled(tmp_path, monkeypa asset_path.write_bytes(b"let a = 2;") expected_hash = hashlib.sha256(b"let a = 2;").hexdigest()[:12] - assert ds.static("demo.js") == f"/-/static/demo.js?_hash={expected_hash}" + assert ds.static("demo.js") == "/-/static/demo.js?_hash={}".format(expected_hash) assert ds.static("demo.js") != first_url @@ -114,12 +114,12 @@ def test_static_hashes_mounted_static_file(tmp_path): expected_hash = hashlib.sha256(b"body { color: black; }").hexdigest()[:12] assert ds.static("styles.css", mount="assets") == ( - f"/assets/styles.css?_hash={expected_hash}" + "/assets/styles.css?_hash={}".format(expected_hash) ) ds._settings["base_url"] = "/prefix/" assert ds.static("styles.css", mount="assets") == ( - f"/prefix/assets/styles.css?_hash={expected_hash}" + "/prefix/assets/styles.css?_hash={}".format(expected_hash) ) @@ -144,7 +144,9 @@ def test_static_hashes_plugin_static_file(tmp_path, monkeypatch): expected_hash = hashlib.sha256(b"console.log('plugin');").hexdigest()[:12] assert ds.static("plugin.js", plugin="datasette_cluster_map") == ( - f"/-/static-plugins/datasette_cluster_map/plugin.js?_hash={expected_hash}" + "/-/static-plugins/datasette_cluster_map/plugin.js?_hash={}".format( + expected_hash + ) ) @@ -165,7 +167,7 @@ def test_static_rejects_path_traversal(tmp_path, monkeypatch): @pytest.mark.asyncio async def test_datasette_constructor(): ds = Datasette() - databases = (await ds.client.get("/-/databases.json")).json()["databases"] + databases = (await ds.client.get("/-/databases.json")).json() assert databases == [ { "name": "_memory", @@ -182,12 +184,11 @@ async def test_datasette_constructor(): @pytest.mark.asyncio async def test_num_sql_threads_zero(): ds = Datasette([], memory=True, settings={"num_sql_threads": 0}) - ds.root_enabled = True db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) await db.execute_write("create table t(id integer primary key)") await db.execute_write("insert into t (id) values (1)") - response = await ds.client.get("/-/threads.json", actor={"id": "root"}) - assert response.json() == {"ok": True, "num_threads": 0, "threads": []} + response = await ds.client.get("/-/threads.json") + assert response.json() == {"num_threads": 0, "threads": []} response2 = await ds.client.get("/test_num_sql_threads_zero/t.json?_shape=array") assert response2.json() == [{"id": 1}] diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 51b38f8d..543077a5 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,7 +1,6 @@ import httpx import pytest import pytest_asyncio - from datasette.app import Datasette @@ -239,7 +238,7 @@ async def test_in_client_returns_false_outside_request(datasette): @pytest.mark.asyncio async def test_in_client_returns_true_inside_request(): """Test that datasette.in_client() returns True inside a client request""" - from datasette import Response, hookimpl + from datasette import hookimpl, Response class TestPlugin: __name__ = "test_in_client_plugin" @@ -319,7 +318,7 @@ async def test_actor_parameter_sets_cookie(datasette): """Passing actor= should sign a ds_actor cookie and authenticate the request.""" response = await datasette.client.get("/-/actor.json", actor={"id": "root"}) assert response.status_code == 200 - assert response.json() == {"ok": True, "actor": {"id": "root"}} + assert response.json() == {"actor": {"id": "root"}} @pytest.mark.asyncio @@ -328,7 +327,7 @@ async def test_actor_parameter_works_with_request_method(datasette): "GET", "/-/actor.json", actor={"id": "root"} ) assert response.status_code == 200 - assert response.json() == {"ok": True, "actor": {"id": "root"}} + assert response.json() == {"actor": {"id": "root"}} @pytest.mark.asyncio @@ -363,7 +362,7 @@ async def test_actor_parameter_merges_with_other_cookies(datasette): cookies={"unrelated": "value"}, ) assert response.status_code == 200 - assert response.json() == {"ok": True, "actor": {"id": "root"}} + assert response.json() == {"actor": {"id": "root"}} @pytest.mark.asyncio diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index e982628b..9c448186 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -1,39 +1,7 @@ +from datasette.utils.asgi import Request import json - import pytest -from datasette.utils.asgi import PayloadTooLarge, Request - - -def _post_scope(headers=None): - return { - "http_version": "1.1", - "method": "POST", - "path": "/", - "raw_path": b"/", - "query_string": b"", - "scheme": "http", - "type": "http", - "headers": headers or [[b"content-type", b"application/json"]], - } - - -def _receive_chunks(chunks): - messages = [ - { - "type": "http.request", - "body": chunk, - "more_body": i < len(chunks) - 1, - } - for i, chunk in enumerate(chunks) - ] - messages.reverse() - - async def receive(): - return messages.pop() - - return receive - @pytest.mark.asyncio async def test_request_post_vars(): @@ -138,70 +106,6 @@ async def test_request_json_invalid(): await request.json() -@pytest.mark.asyncio -async def test_request_post_body_multiple_chunks(): - request = Request(_post_scope(), _receive_chunks([b"hello ", b"world"])) - assert await request.post_body() == b"hello world" - - -@pytest.mark.asyncio -async def test_request_post_body_content_length_too_large(): - # Should reject based on content-length without reading the body - async def receive(): - raise AssertionError("receive() should not be called") - - scope = _post_scope( - headers=[ - [b"content-type", b"application/json"], - [b"content-length", b"101"], - ] - ) - request = Request(scope, receive) - with pytest.raises(PayloadTooLarge): - await request.post_body(max_bytes=100) - - -@pytest.mark.asyncio -async def test_request_post_body_streaming_too_large(): - # No content-length header - limit enforced as chunks arrive - chunks = [b"a" * 60, b"b" * 60, b"c" * 60] - request = Request(_post_scope(), _receive_chunks(chunks)) - with pytest.raises(PayloadTooLarge): - await request.post_body(max_bytes=100) - - -@pytest.mark.asyncio -async def test_request_post_body_limit_from_constructor(): - request = Request( - _post_scope(), _receive_chunks([b"too much data"]), max_post_body_bytes=5 - ) - with pytest.raises(PayloadTooLarge): - await request.post_body() - - -@pytest.mark.asyncio -async def test_request_post_body_limit_disabled(): - body = b"a" * (3 * 1024 * 1024) - request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=0) - assert await request.post_body() == body - - -@pytest.mark.asyncio -async def test_request_post_body_default_limit(): - # Bodies over 2MB are rejected by default - request = Request(_post_scope(), _receive_chunks([b"a" * (2 * 1024 * 1024 + 1)])) - with pytest.raises(PayloadTooLarge): - await request.post_body() - - -@pytest.mark.asyncio -async def test_request_json_too_large(): - body = json.dumps({"rows": ["x" * 100]}).encode("utf-8") - request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=50) - with pytest.raises(PayloadTooLarge): - await request.json() - - def test_request_args(): request = Request.fake("/foo?multi=1&multi=2&single=3") assert "1" == request.args.get("multi") diff --git a/tests/test_internals_response.py b/tests/test_internals_response.py index aa3e1ae2..820b20b2 100644 --- a/tests/test_internals_response.py +++ b/tests/test_internals_response.py @@ -1,8 +1,5 @@ -import json - -import pytest - from datasette.utils.asgi import Response +import pytest def test_response_html(): @@ -55,26 +52,3 @@ async def test_response_set_cookie(): }, {"type": "http.response.body", "body": b""}, ] == events - - -def test_response_error_single_message(): - response = Response.error("Method not allowed", 405) - assert response.status == 405 - assert response.content_type == "application/json; charset=utf-8" - assert json.loads(response.body) == { - "ok": False, - "error": "Method not allowed", - "errors": ["Method not allowed"], - "status": 405, - } - - -def test_response_error_message_list_and_default_status(): - response = Response.error(["First problem", "Second problem"]) - assert response.status == 400 - assert json.loads(response.body) == { - "ok": False, - "error": "First problem; Second problem", - "errors": ["First problem", "Second problem"], - "status": 400, - } diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 50c61995..24fa745d 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -1,7 +1,6 @@ -import pytest - from datasette.app import Datasette from datasette.utils import PrefixedUrlString +import pytest @pytest.fixture(scope="module") diff --git a/tests/test_label_column_for_table.py b/tests/test_label_column_for_table.py index b67b8882..7667b595 100644 --- a/tests/test_label_column_for_table.py +++ b/tests/test_label_column_for_table.py @@ -1,7 +1,6 @@ import pytest - -from datasette.app import Datasette from datasette.database import Database +from datasette.app import Datasette @pytest.mark.asyncio diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py deleted file mode 100644 index 3655285e..00000000 --- a/tests/test_lifespan.py +++ /dev/null @@ -1,259 +0,0 @@ -""" -Tests for wiring Datasette startup (setup_db table counts + invoke_startup) -into the ASGI lifespan protocol. - -These exercise Datasette._startup_sequence() via three different callers: -- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request) -- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan - events (this is what DatasetteClient / plain httpx.ASGITransport uses) -- Both at once, to prove startup hooks run at most once -""" - -import asyncio -import contextlib -import sqlite3 - -import httpx -import pytest - -from datasette import hookimpl -from datasette.app import Datasette -from datasette.database import Database -from datasette.plugins import pm - - -async def _drive_lifespan_startup(app): - """Send a single lifespan.startup message into app's ASGI lifespan loop - and return the list of messages sent back - without ever sending - lifespan.shutdown. Mirrors what a real server does: after startup - completes it parks waiting for the next event. We cancel that wait - once we've observed the startup response, rather than closing the - Datasette instance down with a shutdown message. - """ - messages_sent = [] - startup_responded = asyncio.Event() - delivered = False - - async def receive(): - nonlocal delivered - if not delivered: - delivered = True - return {"type": "lifespan.startup"} - # No further messages: block until the task is cancelled below, - # same as a real server parked waiting for lifespan.shutdown. - await asyncio.Event().wait() - - async def send(message): - messages_sent.append(message) - startup_responded.set() - - task = asyncio.create_task(app({"type": "lifespan"}, receive, send)) - try: - await asyncio.wait_for(startup_responded.wait(), timeout=5) - finally: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - return messages_sent - - -@pytest.mark.asyncio -async def test_lifespan_startup_runs_before_any_request(): - ds = Datasette(memory=True) - assert ds._startup_invoked is False - app = ds.app() - - messages = await _drive_lifespan_startup(app) - - assert {"type": "lifespan.startup.complete"} in messages - assert ds._startup_invoked is True - # Internal catalog tables should be populated too, entirely without an - # HTTP request having been made. - internal_db = ds.get_internal_database() - databases = await internal_db.execute("select * from catalog_databases") - assert len(databases.rows) >= 1 - - -@pytest.mark.asyncio -async def test_lifespan_startup_failure_reports_lifespan_startup_failed(): - class RaisingStartupPlugin: - __name__ = "RaisingStartupPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - raise RuntimeError("boom from startup hook") - - return inner - - ds = Datasette(memory=True) - pm.register(RaisingStartupPlugin(), name="raising_startup_plugin") - try: - app = ds.app() - messages = await _drive_lifespan_startup(app) - finally: - pm.unregister(name="raising_startup_plugin") - - assert messages == [ - {"type": "lifespan.startup.failed", "message": "boom from startup hook"} - ] - # The exception happened before invoke_startup() got to the end of its - # body, so startup is not considered to have completed. - assert ds._startup_invoked is False - - -@pytest.mark.asyncio -async def test_startup_runs_exactly_once_across_lifespan_and_first_request(): - call_count = {"n": 0} - - class CountingStartupPlugin: - __name__ = "CountingStartupPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - call_count["n"] += 1 - - return inner - - ds = Datasette(memory=True) - pm.register(CountingStartupPlugin(), name="counting_startup_plugin") - try: - # Build the ASGI app once, the way a real deployment does - and - # reuse the SAME app instance for both the lifespan drive and the - # HTTP requests below, since a fresh ds.app() call would reset the - # AsgiRunOnFirstRequest fallback's state. - app = ds.app() - - messages = await _drive_lifespan_startup(app) - assert {"type": "lifespan.startup.complete"} in messages - assert call_count["n"] == 1 - - # A first HTTP request (as if the host never sent lifespan events, - # or lifespan already ran) should not run the hook again. - transport = httpx.ASGITransport(app=app) - async with httpx.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - response1 = await client.get("/-/versions.json") - assert response1.status_code == 200 - # ... nor should a second, repeat request. - response2 = await client.get("/-/versions.json") - assert response2.status_code == 200 - finally: - pm.unregister(name="counting_startup_plugin") - - assert call_count["n"] == 1 - - -@pytest.mark.asyncio -async def test_no_lifespan_first_request_still_triggers_startup(): - # Pin today's behavior: a client that never drives ASGI lifespan events - # at all (like httpx.ASGITransport, which DatasetteClient uses) still - # gets startup armed by the AsgiRunOnFirstRequest fallback. - ds = Datasette(memory=True) - assert ds._startup_invoked is False - app = ds.app() - transport = httpx.ASGITransport(app=app) - async with httpx.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - response = await client.get("/-/versions.json") - assert response.status_code == 200 - - assert ds._startup_invoked is True - internal_db = ds.get_internal_database() - databases = await internal_db.execute("select * from catalog_databases") - assert len(databases.rows) >= 1 - - -@pytest.mark.asyncio -async def test_datasette_client_first_request_triggers_startup(): - # Same as above, but through the real DatasetteClient (ds.client) that - # plugins and tests actually use, to confirm nothing regressed there. - ds = Datasette(memory=True) - assert ds._startup_invoked is False - response = await ds.client.get("/-/versions.json") - assert response.status_code == 200 - assert ds._startup_invoked is True - - -@pytest.mark.asyncio -async def test_concurrent_first_requests_all_wait_for_slow_startup(): - call_count = {"n": 0} - - class SlowStartupPlugin: - __name__ = "SlowStartupPlugin" - - @hookimpl - def startup(self, datasette): - async def inner(): - call_count["n"] += 1 - await asyncio.sleep(0.2) - - return inner - - ds = Datasette(memory=True) - pm.register(SlowStartupPlugin(), name="slow_startup_plugin") - try: - app = ds.app() - transport = httpx.ASGITransport(app=app) - async with httpx.AsyncClient( - transport=transport, base_url="http://localhost" - ) as client: - responses = await asyncio.gather( - *[client.get("/-/versions.json") for _ in range(10)] - ) - finally: - pm.unregister(name="slow_startup_plugin") - - # Every one of the 10 simultaneous first requests must have blocked - # until startup actually finished, not raced ahead of it. - assert all(response.status_code == 200 for response in responses) - assert call_count["n"] == 1 - assert ds._startup_invoked is True - - -@pytest.mark.asyncio -async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch): - # Regression test: `datasette serve` (cli.py _serve_async) calls - # ds.invoke_startup() directly, before uvicorn ever sends a - # lifespan.startup event that drives _startup_sequence(). If - # _startup_sequence()'s fast path only checked `_startup_invoked`, it - # would see startup already done and skip the immutable-database - # table-count precompute (setup_db) entirely - a silent regression - # versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally - # on request #1. - db_path = tmp_path / "immutable.db" - conn = sqlite3.connect(str(db_path)) - conn.execute("create table t (id integer primary key)") - conn.commit() - conn.close() - - ds = Datasette([], immutables=[str(db_path)]) - - call_count = {"n": 0} - original_table_counts = Database.table_counts - - async def counting_table_counts(self, *args, **kwargs): - call_count["n"] += 1 - return await original_table_counts(self, *args, **kwargs) - - monkeypatch.setattr(Database, "table_counts", counting_table_counts) - - # Simulate the CLI path: invoke_startup() runs directly and completes - # BEFORE _startup_sequence() ever gets a chance to run setup_db. - await ds.invoke_startup() - assert ds._startup_invoked is True - assert call_count["n"] == 0 - - # The lifespan/first-request path (or the CLI itself, per the fix) - # calling the shared entry point afterwards must still precompute - # table counts for immutable databases. - await ds._startup_sequence() - assert call_count["n"] == 1 - assert ds._setup_db_done is True - - # Idempotency: a second call must not recompute. - await ds._startup_sequence() - assert call_count["n"] == 1 diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py index 61cdb3e0..cdadb091 100644 --- a/tests/test_load_extensions.py +++ b/tests/test_load_extensions.py @@ -1,8 +1,6 @@ -from pathlib import Path - -import pytest - from datasette.app import Datasette +import pytest +from pathlib import Path # not necessarily a full path - the full compiled path looks like "ext.dylib" # or another suffix, but sqlite will, under the hood, decide which file diff --git a/tests/test_messages.py b/tests/test_messages.py index 60eb7938..62d9f647 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -1,6 +1,5 @@ -import pytest - from .utils import cookie_was_deleted +import pytest @pytest.mark.asyncio diff --git a/tests/test_multipart.py b/tests/test_multipart.py index ab38bfb7..0dc3ecd7 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -6,12 +6,12 @@ Uses TDD approach - these tests are written first, then implementation follows. import base64 import json +import pytest from collections import namedtuple -import pytest from multipart_form_data_conformance import get_tests_dir -from datasette.utils.asgi import BadRequest, Request +from datasette.utils.asgi import Request, BadRequest def make_receive(body: bytes): diff --git a/tests/test_package.py b/tests/test_package.py index 43b20589..f05f3ece 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,11 +1,9 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import os import pathlib -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli class CaptureDockerfile: diff --git a/tests/test_permission_endpoints.py b/tests/test_permission_endpoints.py index c54bfbfd..e25be23e 100644 --- a/tests/test_permission_endpoints.py +++ b/tests/test_permission_endpoints.py @@ -6,7 +6,6 @@ Tests for permission endpoints: import pytest import pytest_asyncio - from datasette.app import Datasette @@ -138,7 +137,9 @@ async def test_allowed_json_pagination(): await ds.refresh_schemas() # Test page 1 - response = await ds.client.get("/-/allowed.json?action=view-table&_size=10&_page=1") + response = await ds.client.get( + "/-/allowed.json?action=view-table&page_size=10&page=1" + ) assert response.status_code == 200 data = response.json() assert data["page"] == 1 @@ -146,7 +147,9 @@ async def test_allowed_json_pagination(): assert len(data["items"]) == 10 # Test page 2 - response = await ds.client.get("/-/allowed.json?action=view-table&_size=10&_page=2") + response = await ds.client.get( + "/-/allowed.json?action=view-table&page_size=10&page=2" + ) assert response.status_code == 200 data = response.json() assert data["page"] == 2 @@ -154,10 +157,10 @@ async def test_allowed_json_pagination(): # Verify items are different between pages response1 = await ds.client.get( - "/-/allowed.json?action=view-table&_size=10&_page=1" + "/-/allowed.json?action=view-table&page_size=10&page=1" ) response2 = await ds.client.get( - "/-/allowed.json?action=view-table&_size=10&_page=2" + "/-/allowed.json?action=view-table&page_size=10&page=2" ) items1 = {(item["parent"], item["child"]) for item in response1.json()["items"]} items2 = {(item["parent"], item["child"]) for item in response2.json()["items"]} @@ -320,7 +323,7 @@ async def test_rules_json_pagination(): # Test basic pagination structure - just verify it returns paginated results response = await ds.client.get( - "/-/rules.json?action=view-table&_size=2&_page=1", + "/-/rules.json?action=view-table&page_size=2&page=1", actor={"id": "root"}, ) assert response.status_code == 200 @@ -433,8 +436,8 @@ async def test_execute_sql_requires_view_database(): A user who has execute-sql permission but not view-database permission should not be able to execute SQL on that database. """ - from datasette import hookimpl from datasette.permissions import PermissionSQL + from datasette import hookimpl class TestPermissionPlugin: __name__ = "TestPermissionPlugin" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 73c44682..8323fe92 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,23 +1,19 @@ import collections -import copy -import json -import re -import time -import urllib -from pprint import pprint - -import pytest -import pytest_asyncio from asgiref.sync import async_to_sync -from bs4 import BeautifulSoup as Soup -from click.testing import CliRunner - from datasette.app import Datasette from datasette.cli import cli from datasette.default_permissions import restrictions_allow_action -from datasette.utils import UNSTABLE_API_MESSAGE - from .fixtures import assert_permissions_checked, make_app_client +from click.testing import CliRunner +from bs4 import BeautifulSoup as Soup +import copy +import json +from pprint import pprint +import pytest_asyncio +import pytest +import re +import time +import urllib @pytest.fixture(scope="module") @@ -460,20 +456,6 @@ async def test_permissions_debug(ds_client, filter_): assert checks == expected_checks -@pytest.mark.asyncio -@pytest.mark.parametrize( - "permissions_debug,expected_status", - ( - (1, 200), - (0, 403), - ), -) -async def test_permissions_debug_numeric_boolean(permissions_debug, expected_status): - ds = Datasette(config={"permissions": {"permissions-debug": permissions_debug}}) - response = await ds.client.get("/-/permissions") - assert response.status_code == expected_status - - @pytest.mark.asyncio @pytest.mark.parametrize( "actor,allow,expected_fragment", @@ -605,7 +587,9 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta ) assert ( response.status == expected_status - ), f"path: {path}, permissions: {permissions}, expected_status: {expected_status}, status: {response.status}" + ), "path: {}, permissions: {}, expected_status: {}, status: {}".format( + path, permissions, expected_status, response.status + ) finally: cascade_app_client.ds.config = previous_config @@ -755,20 +739,13 @@ async def test_actor_restricted_permissions( "path": expected_path, } expected = { - "ok": True, - "unstable": UNSTABLE_API_MESSAGE, "action": permission, "allowed": expected_result, "resource": expected_resource, } if actor.get("id"): expected["actor_id"] = actor["id"] - data = response.json() - for key, value in expected.items(): - assert data[key] == value - assert data["actor"] == actor - assert data["explanation"]["allowed"] is expected_result - assert data["explanation"]["summary"] + assert response.json() == expected PermConfigTestCase = collections.namedtuple( @@ -1138,7 +1115,7 @@ def test_cli_create_token(options, expected): ], ) assert 0 == result2.exit_code, result2.output - assert json.loads(result2.output) == {"ok": True, "actor": expected} + assert json.loads(result2.output) == {"actor": expected} _visible_tables_re = re.compile(r">\/((\w+)\/(\w+))\.json<\/a> - Get rows for") @@ -1754,8 +1731,6 @@ async def test_permission_check_view_requires_debug_permission(): data = response.json() assert data["action"] == "view-instance" assert data["allowed"] is True - assert data["explanation"]["allowed"] is True - assert data["explanation"]["summary"] @pytest.mark.asyncio @@ -1781,211 +1756,6 @@ async def test_permission_check_view_query_actions(action): } -@pytest.mark.asyncio -async def test_permission_check_explains_specificity_for_hypothetical_actor(): - ds = Datasette( - config={ - "permissions": {"view-table": {"id": "alice"}}, - "databases": { - "analytics": { - "permissions": {"view-table": False}, - "tables": { - "public": {"permissions": {"view-table": {"id": "alice"}}} - }, - } - }, - } - ) - ds.root_enabled = True - await ds.invoke_startup() - - def path_for(child): - return "/-/check.json?" + urllib.parse.urlencode( - { - "action": "view-table", - "parent": "analytics", - "child": child, - "actor": json.dumps({"id": "alice"}), - } - ) - - public_response = await ds.client.get(path_for("public"), actor={"id": "root"}) - assert public_response.status_code == 200 - public = public_response.json() - assert public["actor"] == {"id": "alice"} - assert public["allowed"] is True - assert public["explanation"]["allowed"] is True - assert public["explanation"]["winning_scope"] == "resource" - public_rules = public["explanation"]["matched_rules"] - assert any( - rule["scope"] == "resource" and rule["effect"] == "allow" and rule["decisive"] - for rule in public_rules - ) - assert any( - rule["scope"] == "parent" - and rule["effect"] == "deny" - and rule["ignored_because"] == "A more specific rule matched" - for rule in public_rules - ) - - private_response = await ds.client.get(path_for("private"), actor={"id": "root"}) - assert private_response.status_code == 200 - private = private_response.json() - assert private["allowed"] is False - assert private["explanation"]["allowed"] is False - assert private["explanation"]["winning_scope"] == "parent" - assert private["explanation"]["summary"].startswith("Denied by a parent-level rule") - - -@pytest.mark.asyncio -async def test_permission_check_explains_deny_wins_at_same_scope(): - ds = Datasette(config={"permissions": {"view-table": {"id": "someone-else"}}}) - ds.root_enabled = True - await ds.invoke_startup() - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "view-table", - "parent": "analytics", - "child": "users", - "actor": json.dumps({"id": "alice"}), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - assert data["explanation"]["winning_scope"] == "global" - rules = data["explanation"]["matched_rules"] - assert any(rule["effect"] == "deny" and rule["decisive"] for rule in rules) - assert any( - rule["effect"] == "allow" - and rule["ignored_because"] == "A deny rule matched at the same scope" - for rule in rules - ) - - -@pytest.mark.asyncio -async def test_permission_check_explains_default_deny(): - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "insert-row", - "parent": "analytics", - "child": "users", - "actor": json.dumps({"id": "alice"}), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - explanation = data["explanation"] - assert explanation["allowed"] is False - assert explanation["matched_rules"] == [] - assert explanation["winning_scope"] is None - assert explanation["summary"] == ( - "Denied because no permission rule matched this actor and resource." - ) - - -@pytest.mark.asyncio -async def test_permission_check_explains_actor_restrictions(): - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - restricted_actor = { - "id": "alice", - "_r": {"r": {"analytics": {"public": ["vt"]}}}, - } - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "view-table", - "parent": "analytics", - "child": "private", - "actor": json.dumps(restricted_actor), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - explanation = data["explanation"] - assert explanation["rule_allowed"] is True - assert explanation["restriction_allowed"] is False - assert explanation["allowed"] is False - assert explanation["restrictions"] - assert any( - restriction["allowed"] is False for restriction in explanation["restrictions"] - ) - assert "actor's restrictions" in explanation["summary"] - - -@pytest.mark.asyncio -async def test_permission_check_explains_required_actions(): - from datasette import hookimpl - from datasette.permissions import PermissionSQL - - class StoreQueryPermissions: - @hookimpl - def permission_resources_sql(self, actor, action): - if not actor or actor.get("id") != "alice": - return None - if action == "store-query": - return PermissionSQL( - sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'alice can store queries' AS reason" - ) - if action == "execute-sql": - return PermissionSQL( - sql="SELECT 'analytics' AS parent, NULL AS child, 0 AS allow, 'alice cannot execute SQL' AS reason" - ) - - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - ds.pm.register(StoreQueryPermissions(), name="store-query-test") - path = "/-/check.json?" + urllib.parse.urlencode( - { - "action": "store-query", - "parent": "analytics", - "actor": json.dumps({"id": "alice"}), - } - ) - response = await ds.client.get(path, actor={"id": "root"}) - assert response.status_code == 200 - data = response.json() - assert data["allowed"] is False - explanation = data["explanation"] - assert explanation["rule_allowed"] is True - assert explanation["required_actions"][0]["action"] == "execute-sql" - assert explanation["required_actions"][0]["allowed"] is False - assert explanation["summary"] == ( - "Denied because store-query also requires execute-sql, which was denied." - ) - - -@pytest.mark.asyncio -async def test_permission_check_hypothetical_actor_validation(): - ds = Datasette() - ds.root_enabled = True - await ds.invoke_startup() - - response = await ds.client.get( - "/-/check.json?action=view-instance&actor=not-json", - actor={"id": "root"}, - ) - assert response.status_code == 400 - assert response.json()["error"].startswith("Invalid actor JSON:") - - response = await ds.client.get( - "/-/check.json?action=view-instance&actor=%5B%5D", - actor={"id": "root"}, - ) - assert response.status_code == 400 - assert response.json()["error"] == "actor must be a JSON object or null" - - @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ @@ -2029,35 +1799,3 @@ async def test_root_allow_block_with_table_restricted_actor(): actor=admin_actor, ) assert result is True - - -@pytest.mark.asyncio -async def test_databases_json_respects_view_database(tmp_path_factory): - # https://github.com/simonw/datasette - /-/databases should not list - # databases the actor is not allowed to view - db_directory = tmp_path_factory.mktemp("dbs") - from datasette.utils import sqlite3 as _sqlite3 - - paths = [] - for name in ("public", "private"): - path = str(db_directory / f"{name}.db") - conn = _sqlite3.connect(path) - conn.execute("vacuum") - conn.close() - paths.append(path) - ds = Datasette( - paths, - config={"databases": {"private": {"allow": {"id": "root"}}}}, - ) - ds.root_enabled = True - await ds.invoke_startup() - try: - anon_response = await ds.client.get("/-/databases.json") - assert anon_response.status_code == 200 - anon_names = {db["name"] for db in anon_response.json()["databases"]} - assert anon_names == {"public"} - root_response = await ds.client.get("/-/databases.json", actor={"id": "root"}) - root_names = {db["name"] for db in root_response.json()["databases"]} - assert root_names == {"public", "private"} - finally: - ds.close() diff --git a/tests/test_playwright.py b/tests/test_playwright.py index eb1edb57..ee396de5 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -1,4 +1,3 @@ -import base64 import json import socket import subprocess @@ -11,10 +10,6 @@ import pytest from datasette.fixtures import write_fixture_database from datasette.utils.sqlite import sqlite3 -PNG_1X1_BYTES = base64.b64decode( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=" -) - def find_free_port(): with socket.socket() as sock: @@ -22,7 +17,7 @@ def find_free_port(): return sock.getsockname()[1] -def wait_for_server(process, url, timeout=30): +def wait_for_server(process, url, timeout=10): deadline = time.monotonic() + timeout last_error = None while time.monotonic() < deadline: @@ -41,20 +36,7 @@ def wait_for_server(process, url, timeout=30): except httpx.HTTPError as ex: last_error = repr(ex) time.sleep(0.1) - if process.poll() is None: - process.terminate() - try: - stdout, stderr = process.communicate(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate() - else: - stdout, stderr = process.communicate() - raise AssertionError( - f"Timed out waiting for {url}: {last_error}\n" - f"stdout:\n{stdout}\n" - f"stderr:\n{stderr}" - ) + raise AssertionError(f"Timed out waiting for {url}: {last_error}") @pytest.fixture @@ -120,22 +102,6 @@ def write_playwright_database(db_path): id integer primary key, created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) ); - create table binary_files ( - id integer primary key, - name text not null, - data blob - ); - create table bulk_defaults ( - id integer primary key, - title text not null, - status text not null default 'todo', - score integer default 5 - ); - create table upsert_items ( - id text primary key, - title text, - metadata text - ); insert into projects (title, metadata, logo, notes, score) values ( 'Build Datasette', @@ -144,22 +110,7 @@ def write_playwright_database(db_path): 'Initial notes', 5 ); - insert into upsert_items (id, title, metadata) values - ('existing', 'Existing title', '{"old": true}'); """) - conn.execute( - "insert into binary_files (name, data) values (?, ?)", - ("Raw bytes", b"\x00\x01\x02\x03"), - ) - conn.execute( - "insert into binary_files (name, data) values (?, ?)", - ("PNG image", PNG_1X1_BYTES), - ) - conn.execute( - "insert into binary_files (name, data) values (?, ?)", - ("Null bytes", None), - ) - conn.commit() finally: conn.close() @@ -172,7 +123,6 @@ def write_playwright_config(config_path): "data": { "permissions": { "create-table": True, - "insert-row": True, "set-column-type": True, }, "tables": { @@ -195,25 +145,6 @@ def write_playwright_config(config_path): "alter-table": True, }, }, - "binary_files": { - "label_column": "name", - "permissions": { - "insert-row": True, - "update-row": True, - "delete-row": True, - }, - }, - "bulk_defaults": { - "permissions": { - "insert-row": True, - }, - }, - "upsert_items": { - "permissions": { - "insert-row": True, - "update-row": True, - }, - }, }, }, }, @@ -347,58 +278,6 @@ def project_row(datasette_server, pk): return rows[0] -def binary_file_blob(datasette_server, pk): - response = httpx.get( - f"{datasette_server}data/binary_files/{pk}.blob", - params={"_blob_column": "data"}, - ) - response.raise_for_status() - return response.content - - -def wait_for_binary_control_file(binary_control, size, name=None): - binary_control.locator( - ".row-edit-binary-size", has_text=f"Binary: {size} bytes" - ).wait_for() - if name: - binary_control.locator(".row-edit-binary-name", has_text=name).wait_for() - - -def bulk_default_rows(datasette_server, **filters): - params = { - "_shape": "objects", - **{key: str(value) for key, value in filters.items()}, - } - response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params) - response.raise_for_status() - return response.json()["rows"] - - -def upsert_item_rows(datasette_server, **filters): - params = { - "_shape": "objects", - **{key: str(value) for key, value in filters.items()}, - } - response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params) - response.raise_for_status() - return response.json()["rows"] - - -def upsert_item_row(datasette_server, pk): - rows = upsert_item_rows(datasette_server, id=pk) - assert len(rows) == 1 - return rows[0] - - -def open_bulk_insert_dialog(page, url): - page.goto(url) - page.locator('button[data-table-action="insert-row"]').click() - dialog = page.locator("#row-edit-dialog") - dialog.wait_for() - dialog.locator(".row-edit-bulk-insert").click() - return dialog - - def open_jump_menu(page): page.keyboard.press("/") page.locator("navigation-search .search-input").wait_for() @@ -502,165 +381,6 @@ def test_create_table_flow(page, datasette_server): assert "NOT NULL DEFAULT 'Untitled'" in schema -@pytest.mark.playwright -def test_create_table_from_data_flow(page, datasette_server): - page.goto(f"{datasette_server}data") - page.locator("details.actions-menu-links summary").click() - page.locator('button[data-database-action="create-table"]').click() - - dialog = page.locator("#table-create-dialog") - dialog.wait_for() - from_data = dialog.locator(".table-create-from-data") - assert from_data.inner_text() == "Create table from data" - from_data.click() - - assert dialog.locator(".table-create-columns").is_hidden() - assert dialog.locator(".table-create-data").is_visible() - assert dialog.locator(".table-create-save").inner_text() == "Preview rows" - assert ( - dialog.locator(".table-create-data-note").inner_text() - == "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea" - ) - assert dialog.locator(".table-create-data-open-file").inner_text() == "open a file" - assert ( - dialog.locator(".table-create-data-editor").evaluate( - """node => Array.from(node.children) - .filter((child) => !child.hidden) - .map((child) => child.className) - .join(" ")""" - ) - == ("table-create-data-note table-create-input table-create-data-textarea") - ) - assert dialog.locator(".table-create-manual").inner_text() == ( - "Create table manually" - ) - assert ( - dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id") - == "table-create-data-textarea" - ) - - textarea = dialog.locator(".table-create-data-textarea") - dropped_value = textarea.evaluate("""node => new Promise((resolve) => { - node.addEventListener("input", () => resolve(node.value), { once: true }); - const file = new File(["short_id,name\\nx,Ada"], "Repo Export 2026!!.CSV", { type: "text/csv" }); - const dataTransfer = new DataTransfer(); - dataTransfer.items.add(file); - node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer })); - node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer })); - node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer })); - })""") - assert dropped_value == "short_id,name\nx,Ada" - assert dialog.locator(".table-create-table-name").input_value() == ( - "repo_export_2026" - ) - - dialog.locator(".table-create-table-name").fill("playwright_from_data") - textarea.fill( - json.dumps( - { - "metadata": [{"short_id": "a", "name": "Ignored", "score": 9}], - "people": [ - {"short_id": "b", "name": "Ada", "score": 1}, - {"short_id": "c", "name": "Bob", "score": 2.5}, - ], - } - ) - ) - dialog.locator(".table-create-save").click() - - assert dialog.locator(".table-create-data-textarea").is_hidden() - assert dialog.locator(".table-create-save").inner_text() == "Create table" - assert dialog.locator(".table-create-cancel").inner_text() == "Back" - assert dialog.locator(".table-create-data-preview-summary").inner_text() == ( - "Previewing 2 rows." - ) - assert dialog.locator(".table-create-data-primary-key").input_value() == "short_id" - preview_text = dialog.locator(".table-create-data-preview-table").inner_text() - assert "short_id" in preview_text - assert "Ada" in preview_text - assert "2.5" in preview_text - preview_cell_style = dialog.locator( - ".table-create-data-preview-table td" - ).first.evaluate( - """node => ({ - overflowWrap: getComputedStyle(node).overflowWrap, - whiteSpace: getComputedStyle(node).whiteSpace - })""" - ) - assert preview_cell_style == { - "overflowWrap": "anywhere", - "whiteSpace": "normal", - } - - dialog.locator(".table-create-cancel").click() - assert dialog.evaluate("node => node.open") - assert dialog.locator(".table-create-data-textarea").is_visible() - assert '"people"' in dialog.locator(".table-create-data-textarea").input_value() - assert dialog.locator(".table-create-save").inner_text() == "Preview rows" - - dialog.locator(".table-create-save").click() - assert dialog.locator(".table-create-save").inner_text() == "Create table" - dialog.locator(".table-create-save").click() - page.wait_for_url("**/data/playwright_from_data") - - response = httpx.get( - f"{datasette_server}data/playwright_from_data.json?_shape=objects" - ) - response.raise_for_status() - data = response.json() - assert data["rows"] == [ - {"short_id": "b", "name": "Ada", "score": 1}, - {"short_id": "c", "name": "Bob", "score": 2.5}, - ] - - -@pytest.mark.playwright -def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank( - page, datasette_server -): - page.goto(f"{datasette_server}data") - page.locator("details.actions-menu-links summary").click() - page.locator('button[data-database-action="create-table"]').click() - - dialog = page.locator("#table-create-dialog") - dialog.wait_for() - dialog.locator(".table-create-from-data").click() - dialog.locator(".table-create-table-name").fill("playwright_numeric_blanks") - dialog.locator(".table-create-data-textarea").fill("name,score\nA,1\nB,") - dialog.locator(".table-create-save").click() - - assert dialog.locator(".table-create-save").inner_text() == "Create table" - preview_text = dialog.locator(".table-create-data-preview-table").inner_text() - assert "A" in preview_text - assert "1" in preview_text - assert "B" in preview_text - assert "null" in preview_text - - dialog.locator(".table-create-save").click() - page.wait_for_url("**/data/playwright_numeric_blanks") - - response = httpx.get( - f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects" - ) - response.raise_for_status() - assert response.json()["rows"] == [ - {"name": "A", "score": 1}, - {"name": "B", "score": None}, - ] - - schema_response = httpx.get( - f"{datasette_server}data/-/query.json", - params={ - "sql": ( - "select type from pragma_table_info('playwright_numeric_blanks') " - "where name = 'score'" - ) - }, - ) - schema_response.raise_for_status() - assert schema_response.json()["rows"] == [{"type": "INTEGER"}] - - @pytest.mark.playwright def test_create_table_foreign_key_selection_updates_column_type(page, datasette_server): page.goto(f"{datasette_server}data") @@ -1081,128 +801,11 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins( @pytest.mark.playwright def test_insert_row_flow_uses_custom_column_field(page, datasette_server): - page.add_init_script(""" - (() => { - let clipboardText = ""; - Object.defineProperty(navigator, "clipboard", { - configurable: true, - get: () => ({ - writeText: async (text) => { - clipboardText = String(text); - }, - readText: async () => clipboardText, - }), - }); - })(); - """) page.goto(f"{datasette_server}data/projects") page.locator('button[data-table-action="insert-row"]').click() dialog = page.locator("#row-edit-dialog") dialog.wait_for() - bulk_insert = dialog.locator(".row-edit-bulk-insert") - bulk_insert.wait_for() - assert bulk_insert.inner_text() == "Insert multiple rows" - current_url = page.url - bulk_insert.click() - assert page.url == current_url - assert dialog.evaluate("node => node.open") - assert dialog.locator(".row-edit-fields").is_hidden() - assert dialog.locator(".row-edit-bulk").is_visible() - assert dialog.locator(".row-edit-save").inner_text() == "Preview rows" - assert ( - dialog.locator(".row-edit-bulk-note").inner_text() - == "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea" - ) - assert dialog.locator(".row-edit-bulk-open-file").inner_text() == "open a file" - assert ( - dialog.locator(".row-edit-bulk-editor").evaluate( - """node => Array.from(node.children) - .filter((child) => !child.hidden) - .map((child) => child.className) - .join(" ")""" - ) - == ( - "row-edit-bulk-note row-edit-input row-edit-bulk-textarea " - "row-edit-bulk-actions" - ) - ) - copy_template = dialog.locator(".row-edit-copy-template") - assert copy_template.inner_text() == "Copy spreadsheet template" - assert ( - dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id") - == "row-edit-bulk-textarea" - ) - assert dialog.locator(".row-edit-copy-template-label-narrow").text_content() == ( - "Copy template" - ) - assert dialog.locator(".row-edit-bulk-template-note").inner_text() == ( - "You can paste the template into Google Sheets or Excel." - ) - assert dialog.locator(".row-edit-bulk-template-note-narrow").text_content() == ( - "Paste into Google Sheets or Excel" - ) - copy_template.click() - page.wait_for_function( - """() => document.querySelector(".row-edit-copy-template").textContent === "Copied" """ - ) - assert page.evaluate("navigator.clipboard.readText()") == ( - "title\tmetadata\tlogo\tnotes\tscore" - ) - textarea = dialog.locator(".row-edit-bulk-textarea") - textarea.fill("title\tmetadata\nFrom TSV\t{}") - assert textarea.input_value() == "title\tmetadata\nFrom TSV\t{}" - dropped_value = textarea.evaluate("""node => new Promise((resolve) => { - node.addEventListener("input", () => resolve(node.value), { once: true }); - const file = new File(["\\n\\ntitle,metadata\\nFrom CSV,{}\\n,\\n , \\n"], "rows.csv", { type: "text/csv" }); - const dataTransfer = new DataTransfer(); - dataTransfer.items.add(file); - node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer })); - node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer })); - node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer })); - })""") - assert dropped_value == "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n" - dialog.locator(".row-edit-save").click() - assert dialog.evaluate("node => node.open") - assert dialog.locator(".row-edit-bulk-textarea").is_hidden() - assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" - assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == ( - "Previewing 1 row." - ) - preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() - assert "id" in preview_text - assert "title" in preview_text - assert "metadata" in preview_text - assert "From CSV" in preview_text - assert dialog.locator(".row-edit-bulk-preview-auto").first.text_content() == "auto" - assert "null" not in preview_text - assert "undefined" not in preview_text - preview_cell_style = dialog.locator( - ".row-edit-bulk-preview-table td" - ).first.evaluate( - """node => ({ - overflowWrap: getComputedStyle(node).overflowWrap, - whiteSpace: getComputedStyle(node).whiteSpace - })""" - ) - assert preview_cell_style == { - "overflowWrap": "anywhere", - "whiteSpace": "normal", - } - assert dialog.locator(".row-edit-cancel").inner_text() == "Back" - dialog.locator(".row-edit-cancel").click() - assert dialog.evaluate("node => node.open") - assert dialog.locator(".row-edit-bulk-textarea").is_visible() - assert dialog.locator(".row-edit-bulk-textarea").input_value() == ( - "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n" - ) - assert dialog.locator(".row-edit-save").inner_text() == "Preview rows" - single_insert = dialog.locator(".row-edit-single-insert") - assert single_insert.inner_text() == "Insert single row" - single_insert.click() - assert dialog.locator(".row-edit-bulk").is_hidden() - assert dialog.locator(".row-edit-fields").is_visible() - assert dialog.locator(".row-edit-save").inner_text() == "Insert row" dialog.locator('input[name="title"]').fill("Launch Datasette Cloud") dialog.locator('textarea[name="metadata"]').fill( '{"ok": false, "source": "playwright"}' @@ -1232,206 +835,6 @@ def test_insert_row_flow_uses_custom_column_field(page, datasette_server): assert data["score"] == 5 -@pytest.mark.playwright -def test_bulk_insert_preview_inserts_rows(page, datasette_server): - page.goto(f"{datasette_server}data/projects") - page.locator('button[data-table-action="insert-row"]').click() - - dialog = page.locator("#row-edit-dialog") - dialog.wait_for() - dialog.locator(".row-edit-bulk-insert").click() - dialog.locator(".row-edit-bulk-textarea").fill( - json.dumps( - { - "metadata": [{"title": "Ignored", "metadata": "{}"}], - "projects": [ - {"title": "Bulk one", "metadata": "{}"}, - {"title": "Bulk two", "metadata": "{}"}, - ], - } - ) - ) - dialog.locator(".row-edit-save").click() - assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" - dialog.locator(".row-edit-save").click() - dialog.locator( - ".row-edit-bulk-progress-status", has_text="2 rows inserted." - ).wait_for() - assert dialog.locator(".row-edit-cancel").inner_text() == "Close and view table" - dialog.locator(".row-edit-cancel").click() - page.wait_for_load_state("domcontentloaded") - assert page.url == f"{datasette_server}data/projects" - - assert project_rows(datasette_server, title="Bulk one") - assert project_rows(datasette_server, title="Bulk two") - - -@pytest.mark.playwright -def test_bulk_insert_upsert_option_updates_existing_and_inserts_new( - page, datasette_server -): - dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/upsert_items") - textarea = dialog.locator(".row-edit-bulk-textarea") - conflict_field = dialog.locator(".row-edit-bulk-conflict") - conflict_select = dialog.locator(".row-edit-bulk-conflict-mode") - - assert conflict_field.is_hidden() - textarea.fill("title\nNo primary key") - assert conflict_field.is_hidden() - - textarea.fill( - "id,title,metadata\nexisting,Updated by upsert,{}\nnew,Inserted by upsert,{}" - ) - assert conflict_field.is_visible() - assert ( - dialog.locator(".row-edit-bulk-conflict-label").inner_text() - == "If the row exists already" - ) - assert conflict_select.input_value() == "ignore" - assert ( - conflict_select.evaluate("""node => Array.from(node.options) - .filter((option) => !option.hidden) - .map((option) => option.textContent.trim())""") - == [ - "Stop with an error", - "Skip existing rows", - "Update existing and insert new", - ] - ) - - conflict_select.select_option("upsert") - assert conflict_select.input_value() == "upsert" - dialog.locator(".row-edit-save").click() - - assert dialog.locator(".row-edit-save").inner_text() == "Update or insert rows" - preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() - assert "Updated by upsert" in preview_text - assert "Inserted by upsert" in preview_text - - dialog.locator(".row-edit-save").click() - dialog.locator( - ".row-edit-bulk-progress-status", has_text="2 rows upserted." - ).wait_for() - - assert upsert_item_row(datasette_server, "existing")["title"] == ( - "Updated by upsert" - ) - assert upsert_item_row(datasette_server, "new")["title"] == "Inserted by upsert" - - -@pytest.mark.playwright -def test_bulk_insert_conflicts_hide_upsert_without_update_permission( - page, datasette_server -): - dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/bulk_defaults") - textarea = dialog.locator(".row-edit-bulk-textarea") - conflict_field = dialog.locator(".row-edit-bulk-conflict") - conflict_select = dialog.locator(".row-edit-bulk-conflict-mode") - - textarea.fill("title\nOnly title") - assert conflict_field.is_hidden() - - textarea.fill("id,title\n1,Only title") - assert conflict_field.is_visible() - assert conflict_select.input_value() == "ignore" - assert ( - conflict_select.evaluate("""node => Array.from(node.options) - .filter((option) => !option.hidden) - .map((option) => option.textContent.trim())""") - == [ - "Stop with an error", - "Skip existing rows", - ] - ) - assert conflict_select.locator('option[value="upsert"]').evaluate( - "node => node.hidden && node.disabled" - ) - - -@pytest.mark.playwright -def test_bulk_insert_live_validation_reports_unknown_columns(page, datasette_server): - dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/projects") - textarea = dialog.locator(".row-edit-bulk-textarea") - error = dialog.locator(".row-edit-error") - save = dialog.locator(".row-edit-save") - - textarea.fill(json.dumps([{"id2": 1, "title": "Unknown column"}])) - error.wait_for() - assert error.inner_text() == "JSON row 1 has unknown column id2." - assert save.is_disabled() - assert textarea.evaluate("node => document.activeElement === node") - - textarea.fill("id2,title\n1,Unknown column") - assert error.inner_text() == "Unknown column id2 in header row." - assert save.is_disabled() - - textarea.fill("[") - assert error.is_hidden() - assert not save.is_disabled() - - textarea.fill(json.dumps([{"id": 1, "title": "Known column"}])) - assert error.is_hidden() - assert not save.is_disabled() - assert dialog.locator(".row-edit-bulk-conflict").is_visible() - assert dialog.locator(".row-edit-bulk-conflict-mode").input_value() == "ignore" - - -@pytest.mark.playwright -def test_bulk_insert_omits_columns_absent_from_pasted_input(page, datasette_server): - page.goto(f"{datasette_server}data/bulk_defaults") - page.locator('button[data-table-action="insert-row"]').click() - - dialog = page.locator("#row-edit-dialog") - dialog.wait_for() - dialog.locator(".row-edit-bulk-insert").click() - dialog.locator(".row-edit-bulk-textarea").fill("title\nOnly title") - dialog.locator(".row-edit-save").click() - - assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" - preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() - assert "Only title" in preview_text - assert "undefined" not in preview_text - assert dialog.locator(".row-edit-bulk-preview-auto").inner_text() == "auto" - - dialog.locator(".row-edit-save").click() - dialog.locator( - ".row-edit-bulk-progress-status", has_text="1 row inserted." - ).wait_for() - - rows = bulk_default_rows(datasette_server, title="Only title") - assert rows == [ - { - "id": 1, - "title": "Only title", - "status": "todo", - "score": 5, - } - ] - - -@pytest.mark.playwright -def test_bulk_insert_preview_accepts_single_column_input(page, datasette_server): - page.goto(f"{datasette_server}data/projects") - page.locator('button[data-table-action="insert-row"]').click() - - dialog = page.locator("#row-edit-dialog") - dialog.wait_for() - dialog.locator(".row-edit-bulk-insert").click() - dialog.locator(".row-edit-bulk-textarea").fill("title\none\ntwo\nthree") - dialog.locator(".row-edit-save").click() - - assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" - assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == ( - "Previewing 3 rows." - ) - preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() - assert "one" in preview_text - assert "two" in preview_text - assert "three" in preview_text - assert "null" not in preview_text - assert "undefined" not in preview_text - - @pytest.mark.playwright def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): page.goto(f"{datasette_server}data/projects") @@ -1439,9 +842,6 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): dialog = page.locator("#row-edit-dialog") dialog.wait_for() - assert dialog.locator(".row-edit-bulk-insert").is_hidden() - assert dialog.locator(".row-edit-single-insert").is_hidden() - assert dialog.locator(".row-edit-bulk").is_hidden() title = dialog.locator('input[name="title"]') title.wait_for() title.fill("Build Datasette, edited") @@ -1477,120 +877,6 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): assert data["notes"] == "Edited from Playwright" -@pytest.mark.playwright -def test_edit_row_binary_control_shows_size_and_image_preview(page, datasette_server): - page.goto(f"{datasette_server}data/binary_files") - page.locator('tr[data-row="1"] button[data-row-action="edit"]').click() - - dialog = page.locator("#row-edit-dialog") - dialog.wait_for() - raw_control = dialog.locator('.row-edit-binary-control[data-column="data"]') - raw_control.wait_for() - assert ( - raw_control.locator(".row-edit-binary-size").inner_text() == "Binary: 4 bytes" - ) - assert raw_control.locator(".row-edit-binary-preview img").count() == 0 - assert dialog.locator('textarea[name="data"]').count() == 0 - assert dialog.locator('input[type="hidden"][name="data"]').count() == 1 - - dialog.locator(".row-edit-cancel").click() - page.locator('tr[data-row="2"] button[data-row-action="edit"]').click() - - image_control = dialog.locator('.row-edit-binary-control[data-column="data"]') - image_control.wait_for() - assert ( - image_control.locator(".row-edit-binary-size").inner_text() - == f"Binary: {len(PNG_1X1_BYTES)} bytes" - ) - image = image_control.locator(".row-edit-binary-preview img") - image.wait_for() - assert image.get_attribute("src").startswith("blob:") - - -@pytest.mark.playwright -def test_edit_row_binary_control_replaces_blob_from_file(page, datasette_server): - replacement = b"Replacement \x00 bytes" - - page.goto(f"{datasette_server}data/binary_files") - page.locator('tr[data-row="1"] button[data-row-action="edit"]').click() - - dialog = page.locator("#row-edit-dialog") - binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') - binary_control.wait_for() - binary_control.locator('input[type="file"]').set_input_files( - { - "name": "replacement.bin", - "mimeType": "application/octet-stream", - "buffer": replacement, - } - ) - wait_for_binary_control_file(binary_control, len(replacement), "replacement.bin") - - dialog.locator(".row-edit-save").click() - page.locator(".row-mutation-status", has_text="Updated row 1").wait_for() - assert binary_file_blob(datasette_server, 1) == replacement - - -@pytest.mark.playwright -def test_edit_row_binary_control_handles_null_blob(page, datasette_server): - replacement = b"From NULL" - - page.goto(f"{datasette_server}data/binary_files") - page.locator('tr[data-row="3"] button[data-row-action="edit"]').click() - - dialog = page.locator("#row-edit-dialog") - binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') - binary_control.wait_for() - assert ( - binary_control.locator(".row-edit-binary-size").inner_text() == "No binary data" - ) - binary_control.locator('input[type="file"]').set_input_files( - { - "name": "from-null.bin", - "mimeType": "application/octet-stream", - "buffer": replacement, - } - ) - wait_for_binary_control_file(binary_control, len(replacement), "from-null.bin") - - dialog.locator(".row-edit-save").click() - page.locator(".row-mutation-status", has_text="Updated row 3").wait_for() - assert binary_file_blob(datasette_server, 3) == replacement - - -@pytest.mark.playwright -def test_insert_row_binary_control_accepts_pasted_file(page, datasette_server): - pasted = b"Pasted \x00 bytes" - - page.goto(f"{datasette_server}data/binary_files") - page.locator('button[data-table-action="insert-row"]').click() - - dialog = page.locator("#row-edit-dialog") - dialog.wait_for() - dialog.locator('input[name="name"]').fill("Pasted bytes") - binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') - binary_control.wait_for() - binary_control.evaluate( - """(node, bytes) => { - const transfer = new DataTransfer(); - transfer.items.add( - new File([new Uint8Array(bytes)], "pasted.bin", { - type: "application/octet-stream" - }) - ); - const event = new Event("paste", { bubbles: true, cancelable: true }); - Object.defineProperty(event, "clipboardData", { value: transfer }); - node.dispatchEvent(event); - }""", - list(pasted), - ) - wait_for_binary_control_file(binary_control, len(pasted), "pasted.bin") - - dialog.locator(".row-edit-save").click() - page.locator(".row-mutation-status", has_text="Inserted row 4").wait_for() - assert binary_file_blob(datasette_server, 4) == pasted - - @pytest.mark.playwright def test_delete_row_flow_removes_row(page, datasette_server): page.goto(f"{datasette_server}data/projects") diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 734f0fc2..da14c714 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,3 +1,21 @@ +from bs4 import BeautifulSoup as Soup +from .fixtures import ( + make_app_client, + TEMP_PLUGIN_SECRET_FILE, + PLUGINS_DIR, + TestClient as _TestClient, +) # noqa +from click.testing import CliRunner +from datasette.app import Datasette +from datasette import cli, hookimpl +from datasette.fixtures import TABLES +from datasette.filters import FilterArguments +from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm +from datasette.permissions import PermissionSQL, Action +from datasette.resources import DatabaseResource +from datasette.utils.sqlite import sqlite3 +from datasette.utils import StartupError, await_me_maybe +from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime import importlib @@ -6,31 +24,8 @@ import os import pathlib import re import textwrap -import urllib - import pytest -from bs4 import BeautifulSoup as Soup -from click.testing import CliRunner -from jinja2 import ChoiceLoader, FileSystemLoader - -from datasette import cli, hookimpl -from datasette.app import Datasette -from datasette.filters import FilterArguments -from datasette.fixtures import TABLES -from datasette.permissions import Action, PermissionSQL -from datasette.plugins import DEFAULT_PLUGINS, get_plugins, pm -from datasette.resources import DatabaseResource -from datasette.utils import StartupError, await_me_maybe -from datasette.utils.sqlite import sqlite3 - -from .fixtures import ( - PLUGINS_DIR, - TEMP_PLUGIN_SECRET_FILE, - make_app_client, -) -from .fixtures import ( - TestClient as _TestClient, -) +import urllib at_memory_re = re.compile(r" at 0x\w+") @@ -40,7 +35,7 @@ at_memory_re = re.compile(r" at 0x\w+") ) def test_plugin_hooks_have_tests(plugin_hook): """Every plugin hook should be referenced in this test module""" - tests_in_this_module = [t for t in globals() if t.startswith("test_hook_")] + tests_in_this_module = [t for t in globals().keys() if t.startswith("test_hook_")] ok = False for test in tests_in_this_module: if plugin_hook in test: @@ -130,11 +125,11 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): response = await ds_client.get(path) assert response.status_code == 200 links = Soup(response.text, "html.parser").find_all("link") - special_href = next( + special_href = [ link for link in links if link.attrs["href"].endswith("/extra-css-urls-demo.css") - )["href"] + ][0]["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] actual_decoded_object = json.loads(base64.b64decode(encoded).decode("utf8")) @@ -157,7 +152,7 @@ async def test_hook_extra_js_urls(ds_client): "type": "module", }, ]: - assert any(s == attrs for s in script_attrs), f"Expected: {attrs}" + assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs) @pytest.mark.asyncio @@ -320,8 +315,7 @@ async def test_plugin_config_env_from_list(ds_client): @pytest.mark.asyncio async def test_plugin_config_file(ds_client): - # Blocking write is fine here - it is tiny test setup, not request handling - with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: # noqa: ASYNC230 + with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -789,13 +783,9 @@ async def test_hook_permission_resources_sql(): @pytest.mark.asyncio async def test_actor_json(ds_client): - assert (await ds_client.get("/-/actor.json")).json() == { - "ok": True, - "actor": None, - } + assert (await ds_client.get("/-/actor.json")).json() == {"actor": None} assert (await ds_client.get("/-/actor.json?_bot2=1")).json() == { - "ok": True, - "actor": {"id": "bot2", "1+1": 2}, + "actor": {"id": "bot2", "1+1": 2} } @@ -829,7 +819,7 @@ def test_hook_register_routes_with_datasette(configured_path): assert response.status_code == 200 assert configured_path.upper() == response.text # Other one should 404 - other_path = next(p for p in ("path1", "path2") if configured_path != p) + other_path = [p for p in ("path1", "path2") if configured_path != p][0] assert client.get(f"/{other_path}/", follow_redirects=True).status_code == 404 @@ -934,7 +924,7 @@ async def test_plugin_startup_can_add_queries(): await datasette.add_query( "data", "from_startup", - f"select {result.first()[0]}", + "select {}".format(result.first()[0]), source="plugin", ) @@ -1046,7 +1036,7 @@ async def test_hook_handle_exception(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async")) async def test_hook_handle_exception_custom_response(ds_client, param): - response = await ds_client.get(f"/trigger-error?{param}=1") + response = await ds_client.get("/trigger-error?{}=1".format(param)) assert response.text == param @@ -1380,7 +1370,7 @@ async def test_hook_register_actions_no_duplicates(duplicate): # This should error: with pytest.raises(StartupError) as ex: await ds.invoke_startup() - assert f"Duplicate action {duplicate}" in str(ex.value) + assert "Duplicate action {}".format(duplicate) in str(ex.value) @pytest.mark.asyncio diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index aebcaa33..6617bc77 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -1,12 +1,10 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import json import os -import textwrap -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli +import textwrap @pytest.mark.serial @@ -72,7 +70,9 @@ def test_publish_cloudrun_prompts_for_service( ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} input-service --max-instances 1", + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format( + tag + ), shell=True, ), ] @@ -107,7 +107,9 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} test --max-instances 1", + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format( + tag + ), shell=True, ), ] @@ -184,13 +186,13 @@ def test_publish_cloudrun_memory_cpu( tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test" expected_call = ( "gcloud run deploy --allow-unauthenticated --platform=managed" - f" --image {tag} test" + " --image {} test".format(tag) ) expected_build_call = f"gcloud builds submit --tag {tag}" if memory: - expected_call += f" --memory {memory}" + expected_call += " --memory {}".format(memory) if cpu: - expected_call += f" --cpu {cpu}" + expected_call += " --cpu {}".format(cpu) if timeout: expected_build_call += f" --timeout {timeout}" # max_instances defaults to 1 diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 4302ed94..cab83654 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,11 +1,9 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import os import pathlib -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli @pytest.mark.serial diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py index 9b17d24b..3af1aace 100644 --- a/tests/test_pytest_autoclose_plugin.py +++ b/tests/test_pytest_autoclose_plugin.py @@ -20,7 +20,6 @@ def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess: cwd=str(tmp_path), capture_output=True, text=True, - check=False, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index 15b7ad0f..b757f221 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -8,36 +8,43 @@ from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage -from datasette.utils import UNSTABLE_API_MESSAGE from datasette.utils.asgi import Forbidden from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" ) -EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" +EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "\n".join( + ( + "create table new_table (", + " id integer primary key,", + " name text", + " -- created text default (datetime('now'))", + ")", + ) +) def _template_option_attributes(html, table): - match = re.search(rf'', + ''.format( + i, i + ), f'', f'', f'', @@ -1028,7 +1027,6 @@ async def test_database_create_table_action_button_and_data(): "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], "defaultExpressions": DEFAULT_EXPRESSION_OPTIONS, - "canInsertRows": False, }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] @@ -1052,40 +1050,6 @@ async def test_database_create_table_action_button_and_data(): ds.close() -@pytest.mark.asyncio -async def test_database_create_table_data_includes_insert_row_permission(): - ds = Datasette( - [], - config={ - "databases": { - "data": { - "permissions": { - "create-table": {"id": "root"}, - "insert-row": {"id": "root"}, - }, - }, - }, - }, - ) - try: - db = ds.add_database( - Database(ds, memory_name="test_database_create_table_insert_permission"), - name="data", - ) - await db.execute_write_script(""" - create table items (id integer primary key, name text); - """) - - response = await ds.client.get("/data", actor={"id": "root"}) - assert response.status_code == 200 - create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[ - "createTable" - ] - assert create_table_data["canInsertRows"] is True - finally: - ds.close() - - @pytest.mark.asyncio async def test_database_create_table_data_includes_custom_column_types(): ds = Datasette( @@ -1352,7 +1316,6 @@ async def test_table_insert_action_button_and_data(): assert insert_data["path"] == "/data/items/-/insert" assert insert_data["tableName"] == "items" assert insert_data["primaryKeys"] == ["id"] - assert insert_data["maxInsertRows"] == 100 assert [column["name"] for column in insert_data["columns"]] == [ "name", "score", @@ -1666,9 +1629,9 @@ async def test_row_update_sets_message(): json={"update": {"name": long_name}, "return": True}, ) assert response.status_code == 200 - assert response.json()["rows"][0]["name"] == long_name + assert response.json()["row"]["name"] == long_name assert ds.unsign(response.cookies["ds_messages"], "messages") == [ - [f"Updated row 1 ({truncated_name})", ds.INFO] + ["Updated row 1 ({})".format(truncated_name), ds.INFO] ] finally: ds.close() @@ -1681,9 +1644,9 @@ def test_table_data_uses_base_url(app_client_base_url_prefix): import re soup = Soup(response.text, "html.parser") - table_script = next( + table_script = [ s for s in soup.find_all("script") if "_datasetteTableData" in (s.string or "") - ) + ][0] match = re.search( r"window\._datasetteTableData\s*=\s*({.*?});", table_script.string, @@ -1711,9 +1674,8 @@ def test_table_fragment_custom_table_include(): @pytest.mark.asyncio async def test_table_fragment_uses_render_cell_hook(): - from markupsafe import Markup - from datasette import hookimpl + from markupsafe import Markup class TestRenderCellPlugin: __name__ = "TestRenderCellPlugin" @@ -1721,7 +1683,7 @@ async def test_table_fragment_uses_render_cell_hook(): @hookimpl def render_cell(self, value, column, table, database): if database == "data" and table == "items" and column == "name": - return Markup(f"{value}") + return Markup("{}".format(value)) return None ds = Datasette(memory=True) @@ -2017,8 +1979,8 @@ async def test_sort_errors(ds_client, json, params, error): assert response.json() == { "ok": False, "error": error, - "errors": [error], "status": 400, + "title": None, } else: assert error in response.text @@ -2260,16 +2222,18 @@ def test_allow_facet_off(allow_facet): ) async def test_format_of_binary_links(size, title, length_bytes): ds = Datasette() - db_name = f"binary-links-{size}" + db_name = "binary-links-{}".format(size) db = ds.add_memory_database(db_name) - sql = f"select zeroblob({size}) as blob" - await db.execute_write(f"create table blobs as {sql}") - response = await ds.client.get(f"/{db_name}/blobs") + sql = "select zeroblob({}) as blob".format(size) + await db.execute_write("create table blobs as {}".format(sql)) + response = await ds.client.get("/{}/blobs".format(db_name)) assert response.status_code == 200 - expected = f"{title}><Binary: {length_bytes} bytes>" + expected = "{}><Binary: {} bytes>".format(title, length_bytes) assert expected in response.text # And test with arbitrary SQL query too - sql_response = await ds.client.get(f"{db_name}/-/query", params={"sql": sql}) + sql_response = await ds.client.get( + "{}/-/query".format(db_name), params={"sql": sql} + ) assert sql_response.status_code == 200 assert expected in sql_response.text @@ -2349,7 +2313,6 @@ async def test_foreign_key_labels_obey_permissions(config): assert root_b.json() == { "ok": True, "next": None, - "next_url": None, "rows": [{"id": 1, "name": "world", "a_id": {"value": 1, "label": "hello"}}], "truncated": False, } @@ -2357,7 +2320,6 @@ async def test_foreign_key_labels_obey_permissions(config): assert anon_b.json() == { "ok": True, "next": None, - "next_url": None, "rows": [{"id": 1, "name": "world", "a_id": 1}], "truncated": False, } diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 691c2d64..7923d0e7 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field import pytest -from datasette.app import TEMPLATE_BASE_CONTEXT, Datasette +from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database from datasette.template_contexts import PAGES, documented_context_keys @@ -40,17 +40,17 @@ def test_documented_fields(): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert ( - context_field.help - ), f"{klass.__name__}.{context_field.name} is missing documentation" + assert context_field.help, "{}.{} is missing documentation".format( + klass.__name__, context_field.name + ) @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_has_docstring_and_documented_template(klass): - assert klass.__doc__, f"{klass.__name__} is missing a docstring" - assert ( - klass.documented_template - ), f"{klass.__name__} is missing a documented_template" + assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) + assert klass.documented_template, "{} is missing a documented_template".format( + klass.__name__ + ) def test_from_extra_documentation_comes_from_the_extra_class(): @@ -105,7 +105,7 @@ def isolate_extra_template_vars_plugins(): # for the rest of the process. The contract documents plugin-free # Datasette core, so unregister any non-default plugin that adds # template variables via the extra_template_vars hook - from datasette.plugins import DEFAULT_PLUGINS, pm + from datasette.plugins import pm, DEFAULT_PLUGINS hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} removed = [] @@ -182,18 +182,18 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - f"Undocumented keys in {page_name} template context: {sorted(undocumented)} - add them to the " - "page's Context class" + "Undocumented keys in {} template context: {} - add them to the " + "page's Context class".format(page_name, sorted(undocumented)) ) assert not no_longer_present, ( - f"Documented keys missing from {page_name} template context: {sorted(no_longer_present)} - this would " - "break custom templates" + "Documented keys missing from {} template context: {} - this would " + "break custom templates".format(page_name, sorted(no_longer_present)) ) def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): - assert doc, f"Base context key {name} is missing docs" + assert doc, "Base context key {} is missing docs".format(name) def test_template_context_docs_cover_every_documented_key(): @@ -201,14 +201,15 @@ def test_template_context_docs_cover_every_documented_key(): assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() for name in TEMPLATE_BASE_CONTEXT: - assert f"``{name}``" in docs, name + assert "``{}``".format(name) in docs, name for page_name, klass in PAGES.items(): title = "{} page".format(klass.__name__.removesuffix("Context")) assert title in docs, title for context_field in klass.documented_fields(): + assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( + context_field.name, page_name + ) assert ( - f"``{context_field.name}``" in docs - ), f"{context_field.name} ({page_name} page)" - assert ( - f"``{context_field.name}`` - ``{context_field.type_name}``" in docs - ), f"{context_field.name} type ({page_name} page)" + "``{}`` - ``{}``".format(context_field.name, context_field.type_name) + in docs + ), "{} type ({} page)".format(context_field.name, page_name) diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 10021ddf..5c87f577 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -2,17 +2,11 @@ Tests for the register_token_handler plugin hook. """ -import pytest - from datasette.app import Datasette from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.tokens import ( - SignedTokenHandler, - TokenHandler, - TokenInvalid, - TokenRestrictions, -) +from datasette.tokens import TokenHandler, TokenRestrictions, SignedTokenHandler +import pytest @pytest.fixture @@ -72,10 +66,10 @@ async def test_verify_token_unknown_returns_none(datasette): @pytest.mark.asyncio -async def test_verify_token_bad_signature_raises(datasette): - """verify_token() should raise TokenInvalid for tokens with bad signatures.""" - with pytest.raises(TokenInvalid): - await datasette.verify_token("dstok_tampered_data_here") +async def test_verify_token_bad_signature_returns_none(datasette): + """verify_token() should return None for tokens with bad signatures.""" + result = await datasette.verify_token("dstok_tampered_data_here") + assert result is None @pytest.mark.asyncio @@ -340,6 +334,5 @@ async def test_signed_tokens_disabled(): ds = Datasette(settings={"allow_signed_tokens": False}) with pytest.raises(ValueError, match="Signed tokens are not enabled"): await ds.create_token("test_actor", handler="signed") - # verify_token should raise TokenInvalid for a dstok_ token - with pytest.raises(TokenInvalid, match="not enabled"): - await ds.verify_token("dstok_anything") + # verify_token should return None rather than raising + assert await ds.verify_token("dstok_anything") is None diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 21cfa952..9db211d3 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,5 +1,4 @@ import pytest - from .fixtures import make_app_client @@ -76,9 +75,10 @@ async def test_trace_child_tasks_resets_contextvar_on_exception(): from datasette import tracer before = tracer.trace_task_id.get() - with pytest.raises(ValueError), tracer.trace_child_tasks(): - assert tracer.trace_task_id.get() is not None - raise ValueError("simulated error") + with pytest.raises(ValueError): + with tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") # The contextvar must be reset even though the block raised assert tracer.trace_task_id.get() == before diff --git a/tests/test_utils.py b/tests/test_utils.py index 1808b3cf..38ebb51e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,17 +2,8 @@ Tests for various datasette helper functions. """ -import hashlib -import json -import os -import pathlib -import tempfile -from unittest.mock import patch - -import pytest - -from datasette import utils from datasette.app import Datasette +from datasette import utils from datasette.utils.asgi import Request from datasette.utils.sqlite import ( sqlite3, @@ -20,6 +11,13 @@ from datasette.utils.sqlite import ( sqlite_table_type, supports_returning, ) +import hashlib +import json +import os +import pathlib +import pytest +import tempfile +from unittest.mock import patch @pytest.mark.parametrize( @@ -134,11 +132,7 @@ def test_path_from_row_pks(row, pks, expected_path): """ {"CategoryID": 1, "Description": "Soft drinks", "Picture": {"$base64": true, "encoded": "FRwCx60F/g=="}} """.strip(), - ), - ( - {"message": b"hello"}, - '{"message": {"$base64": true, "encoded": "aGVsbG8="}}', - ), + ) ], ) def test_custom_json_encoder(obj, expected): @@ -196,7 +190,7 @@ def test_validate_sql_select_good(good_sql): @pytest.mark.parametrize("open_quote,close_quote", [('"', '"'), ("[", "]")]) def test_detect_fts(open_quote, close_quote): - sql = f""" + sql = """ CREATE TABLE "Dumb_Table" ( "TreeID" INTEGER, "qSpecies" TEXT @@ -211,9 +205,9 @@ def test_detect_fts(open_quote, close_quote): "qCaretaker" TEXT ); CREATE VIEW Test_View AS SELECT * FROM Dumb_Table; - CREATE VIRTUAL TABLE {open_quote}Street_Tree_List_fts{close_quote} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open_quote}Street_Tree_List{close_quote}); + CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open}Street_Tree_List{close}); CREATE VIRTUAL TABLE r USING rtree(a, b, c); - """ + """.format(open=open_quote, close=close_quote) conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) assert None is utils.detect_fts(conn, "Dumb_Table") @@ -264,8 +258,8 @@ def test_escape_sqlite_prevents_injection(): conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") malicious = "users] UNION SELECT password FROM users--" - conn.execute(f'CREATE TABLE "{malicious}" (id INTEGER)') - sql = f"select count(*) from {utils.escape_sqlite(malicious)}" + conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) + sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) results = conn.execute(sql).fetchall() conn.close() # The injected UNION must not execute - only the empty malicious table @@ -275,16 +269,16 @@ def test_escape_sqlite_prevents_injection(): @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): - sql = f""" + sql = """ CREATE TABLE [{table}] ( "TreeID" INTEGER, "qSpecies" TEXT ); CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 ("qSpecies", content="{table}"); - """ + """.format(table=table) conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) - assert f"{table}_fts" == utils.detect_fts(conn, table) + assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table) conn.close() @@ -692,6 +686,7 @@ def test_resolve_env_secrets(config, expected): [ ({"id": "blah"}, "blah"), ({"id": "blah", "login": "l"}, "l"), + ({"id": "blah", "login": "l"}, "l"), ({"id": "blah", "login": "l", "username": "u"}, "u"), ({"login": "l", "name": "n"}, "n"), ( @@ -757,21 +752,6 @@ def test_parse_metadata(content, expected): ("select 1 + :one + :two", ["one", "two"]), ("select 'bob' || '0:00' || :cat", ["cat"]), ("select this is invalid :one, :two, :three", ["one", "two", "three"]), - # A string literal containing a comment marker should not hide - # parameters that come after it - ("select * from t where note = '-- TODO' and id = :id", ["id"]), - ("select '--' || :y", ["y"]), - ("select * from t where note = '/* x */' and id = :id", ["id"]), - # Parameters that live inside a comment should be ignored - ("select :x -- and :ignored", ["x"]), - ("select :x /* and :ignored */ from t", ["x"]), - ("select :x /* and :ignored", ["x"]), - # Parameters inside quoted identifiers should be ignored - ("select [a:b] from t where id = :id", ["id"]), - ("select `a:b` from t where id = :id", ["id"]), - ("select `a``:b` from t where id = :id", ["id"]), - # Parameters inside a string literal should be ignored - ("select ':ignored' || :real", ["real"]), ), ) @pytest.mark.parametrize("use_async_version", (False, True)) diff --git a/tests/test_utils_check_callable.py b/tests/test_utils_check_callable.py index 857b73cd..4f72f9ff 100644 --- a/tests/test_utils_check_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,6 +1,5 @@ -import pytest - from datasette.utils.check_callable import check_callable +import pytest class AsyncClass: diff --git a/tests/test_utils_permissions.py b/tests/test_utils_permissions.py index 918dab95..bc3599c2 100644 --- a/tests/test_utils_permissions.py +++ b/tests/test_utils_permissions.py @@ -1,17 +1,14 @@ -from collections.abc import Callable - import pytest - from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.utils.permissions import resolve_permissions_from_catalog +from typing import Callable, List @pytest.fixture def db(): ds = Datasette() import tempfile - from datasette.database import Database path = tempfile.mktemp(suffix="demo.db") @@ -130,7 +127,7 @@ def plugin_root_deny_for_all() -> Callable[[str], PermissionSQL]: def plugin_conflicting_same_child_rules( user: str, parent: str, child: str -) -> list[Callable[[str], PermissionSQL]]: +) -> List[Callable[[str], PermissionSQL]]: def allow_provider(action: str) -> PermissionSQL: return PermissionSQL( """ @@ -280,7 +277,9 @@ async def test_alice_global_allow_with_specific_denies_catalog(db): # Alice can see everything except accounting/sales and hr/* assert "/accounting/sales" in res_denied(rows) for r in rows: - if r["parent"] == "hr" or r["resource"] == "/accounting/sales": + if r["parent"] == "hr": + assert r["allow"] == 0 + elif r["resource"] == "/accounting/sales": assert r["allow"] == 0 else: assert r["allow"] == 1 diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index a6f95e5b..979ff9e1 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -1,7 +1,7 @@ import pytest -from datasette.utils.sql_analysis import analyze_sql_tables from datasette.utils.sqlite import sqlite3 +from datasette.utils.sql_analysis import analyze_sql_tables @pytest.fixture diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 66599c54..88ce5520 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -3,16 +3,14 @@ Tests for the write_wrapper plugin hook. """ import asyncio -import sqlite3 -import time from dataclasses import dataclass - -import pytest - from datasette.app import Datasette from datasette.events import Event from datasette.hookspecs import hookimpl from datasette.plugins import pm +import pytest +import sqlite3 +import time @dataclass @@ -115,8 +113,7 @@ async def test_write_wrapper_exception_thrown_into_generator(datasette): def wrapper(conn): try: yield - except Exception as e: # noqa: BLE001 - # Test helper deliberately captures whatever the wrapped write raised + except Exception as e: caught["error"] = e return wrapper @@ -235,6 +232,7 @@ async def test_write_wrapper_return_none_skips(datasette): @hookimpl def write_wrapper(datasette, database, request, transaction): log.append("hook-called") + return None pm.register(Plugin(), name="test_skip") try: @@ -341,7 +339,7 @@ async def test_write_wrapper_via_api(tmp_path): "/test/api_test/-/insert", json={"row": {"name": "test"}, "return": True}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", }, ) @@ -468,7 +466,7 @@ async def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny try: request = FakeRequest(actor) if should_deny: - with pytest.raises(sqlite3.DatabaseError, match="not authorized"): + with pytest.raises(Exception): await db.execute_write_fn( lambda conn: conn.execute( f"insert into {table} (value) values ('test')"
{i}{}{i}{i}a{i}