diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml new file mode 100644 index 00000000..fdbc71c9 --- /dev/null +++ b/.github/actions/setup-sqlite-version/action.yml @@ -0,0 +1,39 @@ +name: "Setup SQLite version" +description: "Build and activate a specific SQLite version from its amalgamation archive" +inputs: + version: + description: "The SQLite version to install" + required: true + cflags: + description: "CFLAGS to use when compiling SQLite" + required: false + default: "" + skip-activate: + description: "Set to true to skip modifying the library path" + required: false + default: "false" + fallback-urls: + description: "Whitespace-separated fallback download URLs to try after sqlite.org" + required: false + default: "" +outputs: + sqlite-location: + description: "Directory containing the compiled SQLite library" + value: ${{ steps.build.outputs.sqlite-location }} +runs: + using: "composite" + steps: + - shell: bash + run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/sqlite-versions/downloads + key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 + - id: build + shell: bash + run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" + env: + SQLITE_VERSION: ${{ inputs.version }} + SQLITE_CFLAGS: ${{ inputs.cflags }} + SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} + SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh new file mode 100644 index 00000000..03d6a68f --- /dev/null +++ b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" +cflags="${SQLITE_CFLAGS:-}" +skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" +extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" + +case "$version_spec" in + 3.46 | 3.46.0) + sqlite_version="3.46.0" + sqlite_year="2024" + amalgamation_id="3460000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" + ;; + 3.25 | 3.25.0) + sqlite_version="3.25.0" + sqlite_year="2018" + amalgamation_id="3250000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1" + ;; + *) + echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." + exit 1 + ;; +esac + +case "$(uname -s)" in + Linux) + library_name="libsqlite3.so.0" + library_path_var="LD_LIBRARY_PATH" + ;; + Darwin) + library_name="libsqlite3.dylib" + library_path_var="DYLD_LIBRARY_PATH" + ;; + *) + echo "::error::Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +runner_temp="${RUNNER_TEMP:-}" +if [ -z "$runner_temp" ]; then + runner_temp="$(mktemp -d)" +fi + +filename="sqlite-amalgamation-${amalgamation_id}" +official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" +download_dir="${runner_temp}/sqlite-versions/downloads" +source_root="${runner_temp}/sqlite-versions/source" +source_dir="${source_root}/${filename}" +build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" +archive_path="${download_dir}/${filename}.zip" + +mkdir -p "$download_dir" "$source_root" "$build_dir" + +download_archive() { + local url + local candidate_path="${archive_path}.tmp" + local urls=("$official_url") + + for url in $builtin_fallback_urls $extra_fallback_urls; do + urls+=("$url") + done + + rm -f "$candidate_path" + for url in "${urls[@]}"; do + echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" + if curl \ + --fail \ + --location \ + --show-error \ + --retry 5 \ + --retry-delay 2 \ + --retry-max-time 180 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 240 \ + --output "$candidate_path" \ + "$url"; then + mv "$candidate_path" "$archive_path" + return 0 + fi + + echo "::warning::Download failed from ${url}" + rm -f "$candidate_path" + done + + echo "::error::Could not download SQLite ${sqlite_version} amalgamation" + return 1 +} + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + if [ ! -f "$archive_path" ]; then + download_archive + fi + + rm -rf "$source_dir" + unzip -q "$archive_path" -d "$source_root" +fi + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" + exit 1 +fi + +read -r -a cflag_args <<< "$cflags" + +echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" +gcc \ + -fPIC \ + -shared \ + "${cflag_args[@]}" \ + "${source_dir}/sqlite3.c" \ + "-I${source_dir}" \ + -o "${build_dir}/${library_name}" + +if [ "$library_name" = "libsqlite3.so.0" ]; then + ln -sf "$library_name" "${build_dir}/libsqlite3.so" +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" +else + echo "sqlite-location=${build_dir}" +fi + +case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in + true | 1 | yes) + echo "Skipping ${library_path_var} activation" + ;; + *) + existing_value="${!library_path_var:-}" + if [ -n "${GITHUB_ENV:-}" ]; then + if [ -n "$existing_value" ]; then + echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" + else + echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" + fi + fi + echo "Added ${build_dir} to ${library_path_var}" + ;; +esac diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b0640ae8..3fc83438 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index b8fb8aaa..00000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Read the Docs Pull Request Preview -on: - pull_request: - types: - - opened - -permissions: - pull-requests: write - -jobs: - documentation-links: - runs-on: ubuntu-latest - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "datasette" diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 5275ddef..f5b8dbf6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -16,7 +16,7 @@ jobs: matrix: browser: [chromium, firefox, webkit] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python 3.14 uses: actions/setup-python@v6 with: @@ -25,14 +25,14 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Cache uv - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/uv key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} restore-keys: | ${{ runner.os }}-py3.14-uv- - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 735e14e9..d92ab82b 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v6 - - uses: actions/cache@v5 + uses: actions/checkout@v7 + - uses: actions/cache@v6 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87300593..21ed4c12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: @@ -56,7 +56,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index e622ef4c..c5a4f0db 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 9a808194..58635025 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 59b5fbc0..ecde5940 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index c514048e..e9bd4bab 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5162c47a..5e81ed82 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +20,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 23fce459..2fdb3a40 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,7 +25,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -34,7 +34,7 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + uses: ./.github/actions/setup-sqlite-version with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e47db6f..751eedfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index a033cd92..f2c074a6 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 72af1eec..5b8818c3 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: diff --git a/Justfile b/Justfile index 5fcd9afd..6ffff870 100644 --- a/Justfile +++ b/Justfile @@ -33,10 +33,11 @@ export DATASETTE_SECRET := "not_a_secret" uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt uv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, ruff, cog +# Run linters: black, ruff, prettier, cog @lint: codespell uv run black datasette tests --check uv run ruff check datasette tests + npm run prettier -- --check uv run cog --check README.md docs/*.rst # Apply ruff fixes diff --git a/datasette/__init__.py b/datasette/__init__.py index eb18e59e..e0022178 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,8 +1,14 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenRestrictions # noqa -from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa +from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa +from datasette.utils.asgi import ( # noqa + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa diff --git a/datasette/app.py b/datasette/app.py index 4366bb08..0e31273d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -92,7 +92,6 @@ from .views.table import ( TableInsertView, TableUpsertView, TableSetColumnTypeView, - TableSetLabelColumnsView, TableDropView, TableFragmentView, table_view, @@ -112,6 +111,7 @@ from .utils import ( baseconv, call_with_supported_arguments, detect_json1, + add_cors_headers, display_actor, escape_css_string, escape_sqlite, @@ -120,7 +120,6 @@ from .utils import ( module_from_path, move_plugins_and_allow, move_table_config, - normalize_label_columns, parse_metadata, resolve_env_secrets, resolve_routes, @@ -132,8 +131,10 @@ from .utils import ( redact_keys, row_sql_params_pks, ) +from .tokens import TokenInvalid from .utils.asgi import ( AsgiLifespan, + BadRequest, Forbidden, NotFound, DatabaseNotFound, @@ -208,6 +209,11 @@ SETTINGS = ( 100, "Maximum rows that can be inserted at a time using the bulk insert API", ), + Setting( + "max_post_body_bytes", + 2 * 1024 * 1024, + "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit", + ), Setting( "num_sql_threads", 3, @@ -747,19 +753,7 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - placeholders = "(?, ?, ?, ?)" - values = [database_name, str(db.path), db.is_memory, schema_version] - if db.path is None: - placeholders = "(?, null, ?, ?)" - values = [database_name, db.is_memory, schema_version] - await internal_db.execute_write( - """ - INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) - VALUES {} - """.format(placeholders), - values, - ) - await populate_schema_tables(internal_db, db) + await populate_schema_tables(internal_db, db, schema_version) @property def urls(self): @@ -832,8 +826,6 @@ class Datasette: await self._save_queries_from_config() # Load column_types from config into internal DB await self._apply_column_types_config() - # Seed label_column config into internal DB (first run only) - await self._apply_label_columns_config() for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) self._startup_invoked = True @@ -909,7 +901,9 @@ class Datasette: Verify an API token by trying all registered token handlers. Returns an actor dict from the first handler that recognizes the - token, or None if no handler accepts it. + token, or None if no handler accepts it. A handler may raise + TokenInvalid for a token it recognizes but rejects (bad signature, + expired) - Datasette turns that into a 401 response. """ for token_handler in self._token_handlers(): result = await token_handler.verify_token(self, token) @@ -1451,60 +1445,6 @@ class Datasette: [database, resource, column], ) - async def get_label_columns(self, database: str, resource: str): - """ - Return the explicit label_column override for this resource, as a - list of column names, or None if no override has been set (in which - case automatic detection should be used). - """ - row = await self.get_internal_database().execute( - "SELECT columns FROM label_columns " - "WHERE database_name = ? AND resource_name = ?", - [database, resource], - ) - rows = row.rows - if not rows: - return None - return json.loads(rows[0][0]) - - async def set_label_columns( - self, database: str, resource: str, columns: list - ) -> None: - """Set the label column(s) for a resource. Overwrites any existing value.""" - await self.get_internal_database().execute_write( - """INSERT OR REPLACE INTO label_columns - (database_name, resource_name, columns) - VALUES (?, ?, ?)""", - [database, resource, json.dumps(columns)], - ) - - async def remove_label_columns(self, database: str, resource: str) -> None: - """Remove the label column(s) override for a resource.""" - await self.get_internal_database().execute_write( - "DELETE FROM label_columns " - "WHERE database_name = ? AND resource_name = ?", - [database, resource], - ) - - async def _apply_label_columns_config(self): - """ - Seed label_column config from datasette.json into the internal DB, - but only the first time a table is seen - once a row exists (whether - from config or from the set-label-columns API) it is left alone on - subsequent startups. - """ - for db_name, db_conf in (self.config or {}).get("databases", {}).items(): - for table_name, table_conf in db_conf.get("tables", {}).items(): - if "label_column" not in table_conf: - continue - columns = normalize_label_columns(table_conf["label_column"]) - await self.get_internal_database().execute_write( - """INSERT INTO label_columns (database_name, resource_name, columns) - VALUES (?, ?, ?) - ON CONFLICT(database_name, resource_name) DO NOTHING""", - [db_name, table_name, json.dumps(columns)], - ) - def get_internal_database(self): return self._internal_database @@ -2183,17 +2123,17 @@ class Datasette: ) if not visible: return {} - label_columns = await db.label_columns_for_table(other_table) - if not label_columns: + label_column = await db.label_column_for_table(other_table) + if not label_column: return {(fk["column"], value): str(value) for value in values} labeled_fks = {} sql = """ - select {other_column}, {label_columns} + select {other_column}, {label_column} from {other_table} where {other_column} in ({placeholders}) """.format( other_column=escape_sqlite(other_column), - label_columns=", ".join(escape_sqlite(column) for column in label_columns), + label_column=escape_sqlite(label_column), other_table=escape_sqlite(other_table), placeholders=", ".join(["?"] * len(set(values))), ) @@ -2202,19 +2142,8 @@ class Datasette: except QueryInterrupted: pass else: - for row in results: - id = row[0] - if len(label_columns) == 1: - labeled_fks[(fk["column"], id)] = row[1] - else: - label_values = [ - str(value) - for value in row[1:] - if value is not None and value != "" - ] - labeled_fks[(fk["column"], id)] = ( - " ".join(label_values) if label_values else str(id) - ) + for id, value in results: + labeled_fks[(fk["column"], id)] = value return labeled_fks def absolute_url(self, request, path): @@ -2237,6 +2166,18 @@ class Datasette: for name, d in self.databases.items() ] + async def _connected_databases_for_actor(self, actor): + page = await self.allowed_resources("view-database", actor) + allowed_names = {resource.parent async for resource in page.all()} + return [ + database + for database in self._connected_databases() + if database["name"] in allowed_names + ] + + async def _databases_data(self, request): + return {"databases": await self._connected_databases_for_actor(request.actor)} + def _versions(self): conn = sqlite3.connect(":memory:") self._prepare_connection(conn, "_memory") @@ -2583,8 +2524,8 @@ class Datasette: def add_route(view, regex): routes.append((regex, view)) - add_route(IndexView.as_view(self), r"/(\.(?Pjsono?))?$") - add_route(IndexView.as_view(self), r"/-/(\.(?Pjsono?))?$") + add_route(IndexView.as_view(self), r"/(\.(?Pjson))?$") + add_route(IndexView.as_view(self), r"/-/(\.(?Pjson))?$") add_route(permanent_redirect("/-/"), r"/-$") add_route(favicon, "/favicon.ico") @@ -2620,7 +2561,10 @@ class Datasette: ) add_route( JsonDataView.as_view( - self, "plugins.json", self._plugins, needs_request=True + self, + "plugins.json", + self._plugins, + needs_request=True, ), r"/-/plugins(\.(?Pjson))?$", ) @@ -2633,11 +2577,18 @@ class Datasette: r"/-/config(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view(self, "threads.json", self._threads), + JsonDataView.as_view( + self, "threads.json", self._threads, permission="permissions-debug" + ), r"/-/threads(\.(?Pjson))?$", ) add_route( - JsonDataView.as_view(self, "databases.json", self._connected_databases), + JsonDataView.as_view( + self, + "databases.json", + self._databases_data, + needs_request=True, + ), r"/-/databases(\.(?Pjson))?$", ) add_route( @@ -2650,7 +2601,7 @@ class Datasette: JsonDataView.as_view( self, "actions.json", - self._actions, + lambda: {"actions": self._actions()}, template="debug_actions.html", permission="permissions-debug", ), @@ -2805,10 +2756,6 @@ class Datasette: TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) - add_route( - TableSetLabelColumnsView.as_view(self), - r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-label-columns$", - ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", @@ -2862,6 +2809,10 @@ class Datasette: db, table_name, _ = await self.resolve_table(request) pk_values = urlsafe_components(request.url_vars["pks"]) sql, params, pks = await row_sql_params_pks(db, table_name, pk_values) + if len(pk_values) != len(pks): + raise BadRequest( + "URL row identifier does not match the primary key for this table" + ) results = await db.execute(sql, params, truncate=True) row = results.first() if row is None: @@ -2920,7 +2871,11 @@ class DatasetteRouter: if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] scope = dict(scope, route_path=path) - request = Request(scope, receive) + request = Request( + scope, + receive, + max_post_body_bytes=self.ds.setting("max_post_body_bytes"), + ) # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -2940,13 +2895,24 @@ class DatasetteRouter: # Handle authentication default_actor = scope.get("actor") or None actor = None + token_error = None results = pm.hook.actor_from_request(datasette=self.ds, request=request) for result in results: - result = await await_me_maybe(result) + try: + result = await await_me_maybe(result) + except TokenInvalid as ex: + # A presented token was recognized but rejected - fail the + # request with a 401 even if another credential is valid, + # but keep awaiting the remaining coroutines first + if token_error is None: + token_error = ex + continue if result and actor is None: actor = result # Don't break — we must await all coroutines to avoid # "coroutine was never awaited" warnings + if token_error is not None: + return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) @@ -2978,6 +2944,15 @@ class DatasetteRouter: except Exception as exception: return await self.handle_exception(request, send, exception) + async def handle_401(self, request, send, exception): + # A presented bearer token was recognized by a handler but rejected. + # Bearer tokens are API credentials, so this is always JSON. + headers = {"www-authenticate": 'Bearer error="invalid_token"'} + if self.ds.cors: + add_cors_headers(headers) + response = Response.error([str(exception)], 401, headers=headers) + await response.asgi_send(send) + async def handle_404(self, request, send, exception=None): # If path contains % encoding, redirect to tilde encoding if "%" in request.path: diff --git a/datasette/database.py b/datasette/database.py index b76c8397..bab3a378 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -17,6 +17,7 @@ from .utils import ( detect_fts, detect_primary_keys, detect_spatialite, + escape_sqlite, get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, @@ -246,6 +247,7 @@ class Database: request=None, return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, + transaction=True, ): self._check_not_closed() if returning_limit < 0: @@ -258,7 +260,9 @@ class Database: ) with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn(_inner, block=block, request=request) + results = await self.execute_write_fn( + _inner, block=block, request=request, transaction=transaction + ) return results async def execute_write_script(self, sql, block=True, request=None): @@ -348,6 +352,7 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: + self._write_connection.execute("BEGIN IMMEDIATE") result = fn(self._write_connection) else: result = fn(self._write_connection) @@ -477,6 +482,7 @@ class Database: try: if task.transaction: with conn: + conn.execute("BEGIN IMMEDIATE") result = task.fn(conn) else: result = task.fn(conn) @@ -603,7 +609,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", + f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})", custom_time_limit=limit, ) ).rows[0][0] @@ -665,10 +671,12 @@ class Database: async def fts_table(self, table): return await self.execute_fn(lambda conn: detect_fts(conn, table)) - async def label_columns_for_table(self, table): - explicit_label_columns = await self.ds.get_label_columns(self.name, table) - if explicit_label_columns is not None: - return explicit_label_columns + async def label_column_for_table(self, table): + explicit_label_column = (await self.ds.table_config(self.name, table)).get( + "label_column" + ) + if explicit_label_column: + return explicit_label_column def column_details(conn): # Returns {column_name: (type, is_unique)} @@ -693,13 +701,13 @@ class Database: if is_unique and type_ is str ] if len(unique_text_columns) == 1: - return [unique_text_columns[0]] + return unique_text_columns[0] column_names = list(column_details.keys()) # Is there a name or title column? name_or_title = [c for c in column_names if c.lower() in ("name", "title")] if name_or_title: - return [name_or_title[0]] + return name_or_title[0] # If a table has two columns, one of which is ID, then label_column is the other one if ( column_names @@ -707,9 +715,9 @@ class Database: and ("id" in column_names or "pk" in column_names) and not set(column_names) == {"id", "pk"} ): - return [c for c in column_names if c not in ("id", "pk")][0:1] + return [c for c in column_names if c not in ("id", "pk")][0] # Couldn't find a label: - return [] + return None async def foreign_keys_for_table(self, table): return await self.execute_fn( diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 0a4b208e..602e0df4 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -61,6 +61,12 @@ def register_actions(): description="Create tables", resource_class=DatabaseResource, ), + Action( + name="create-view", + abbr="cv", + description="Create views", + resource_class=DatabaseResource, + ), Action( name="store-query", abbr="sq", @@ -105,18 +111,18 @@ def register_actions(): description="Set column type", resource_class=TableResource, ), - Action( - name="set-label-columns", - abbr="slc", - description="Set label column(s)", - resource_class=TableResource, - ), Action( name="drop-table", abbr="dt", description="Drop tables", resource_class=TableResource, ), + Action( + name="drop-view", + abbr="dv", + description="Drop views", + resource_class=TableResource, + ), # Query-level actions (child-level) Action( name="view-query", diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index aab87c1c..8edc976e 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -96,6 +96,10 @@ class ConfigPermissionProcessor: """Evaluate an allow block against the current actor.""" if allow_block is None: return None + # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as + # integers, but should retain the CLI's boolean 1/0 behavior. + if isinstance(allow_block, int) and allow_block in (0, 1): + return bool(allow_block) return actor_matches_allow(self.actor, allow_block) def is_in_restriction_allowlist( diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py index cfead5ba..e41434ef 100644 --- a/datasette/default_table_actions.py +++ b/datasette/default_table_actions.py @@ -6,41 +6,24 @@ from datasette.resources import TableResource def table_actions(datasette, actor, database, table, request): async def inner(): db = datasette.get_database(database) - actions = [] - if db.is_mutable and await datasette.allowed( + if not db.is_mutable: + return [] + if not await datasette.allowed( action="alter-table", resource=TableResource(database=database, table=table), actor=actor, ): - actions.append( - { - "type": "button", - "label": "Alter table", - "description": "Change columns and primary key for this table.", - "attrs": { - "aria-label": "Alter table {}".format(table), - "data-table-action": "alter-table", - }, - } - ) - # Not gated on db.is_mutable - label configuration is a display - # preference stored in the internal DB, not a schema change. - if await datasette.allowed( - action="set-label-columns", - resource=TableResource(database=database, table=table), - actor=actor, - ): - actions.append( - { - "type": "button", - "label": "Set label column(s)", - "description": "Choose which column(s) are used to label this table's rows.", - "attrs": { - "aria-label": "Set label columns for {}".format(table), - "data-table-action": "set-label-columns", - }, - } - ) - return actions + return [] + return [ + { + "type": "button", + "label": "Alter table", + "description": "Change columns and primary key for this table.", + "attrs": { + "aria-label": "Alter table {}".format(table), + "data-table-action": "alter-table", + }, + } + ] return inner diff --git a/datasette/extras.py b/datasette/extras.py index 36014185..fb8c2e06 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -5,6 +5,8 @@ from typing import ClassVar from asyncinject import Registry +from datasette.utils.asgi import BadRequest + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -113,6 +115,17 @@ class ExtraRegistry: self._allowed_names[key] = names return names + def validate_requested(self, requested, scope): + """ + Raise BadRequest if any requested extra name is not a public extra + for this scope. Used by data formats such as .json - HTML pages + silently ignore unknown names instead. + """ + allowed = self._allowed_names_for_scope(scope, include_internal=False) + unknown = sorted(name for name in requested if name not in allowed) + if unknown: + raise BadRequest("Unknown _extra: {}".format(", ".join(unknown))) + async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..5f757df3 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -85,7 +85,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table - self.sql = sql or f"select * from [{table}]" + self.sql = sql or f"select * from {escape_sqlite(table)}" self.params = params or [] self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 41c48396..91b1ff96 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,9 +1,19 @@ from datasette import hookimpl, Response +from .utils import add_cors_headers @hookimpl(trylast=True) def forbidden(datasette, request, message): async def inner(): + if ( + request.path.split("?")[0].endswith(".json") + or "application/json" in (request.headers.get("accept") or "") + or request.headers.get("content-type") == "application/json" + ): + headers = {} + if datasette.cors: + add_cors_headers(headers) + return Response.error(message, 403, headers=headers) return Response.html( await datasette.render_template( "error.html", diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 2b311644..e255ddf2 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,5 +1,5 @@ from datasette import hookimpl, Response -from .utils import add_cors_headers +from .utils import add_cors_headers, error_body from .utils.asgi import ( Base400, ) @@ -28,6 +28,7 @@ def handle_exception(datasette, request, exception): rich.get_console().print_exception(show_locals=True) title = None + plain_message = None if isinstance(exception, Base400): status = exception.status info = {} @@ -36,6 +37,7 @@ def handle_exception(datasette, request, exception): status = exception.status info = exception.error_dict message = exception.message + plain_message = exception.plain_message if exception.message_is_html: message = Markup(message) title = exception.title @@ -45,6 +47,13 @@ def handle_exception(datasette, request, exception): message = str(exception) traceback.print_exc() templates = [f"{status}.html", "error.html"] + headers = {} + if datasette.cors: + add_cors_headers(headers) + if request.path.split("?")[0].endswith(".json"): + body = dict(info) + body.update(error_body(plain_message or message, status)) + return Response.json(body, status=status, headers=headers) info.update( { "ok": False, @@ -53,24 +62,18 @@ def handle_exception(datasette, request, exception): "title": title, } ) - headers = {} - if datasette.cors: - add_cors_headers(headers) - if request.path.split("?")[0].endswith(".json"): - return Response.json(info, status=status, headers=headers) - else: - environment = datasette.get_jinja_environment(request) - template = environment.select_template(templates) - return Response.html( - await template.render_async( - dict( - info, - urls=datasette.urls, - menu_links=lambda: [], - ) - ), - status=status, - headers=headers, - ) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) + return Response.html( + await template.render_async( + dict( + info, + urls=datasette.urls, + menu_links=lambda: [], + ) + ), + status=status, + headers=headers, + ) return inner diff --git a/datasette/renderer.py b/datasette/renderer.py index f40e3dbb..7c94f6ee 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,6 +1,7 @@ import json from datasette.extras import extra_names_from_request from datasette.utils import ( + error_body, value_as_boolean, remove_infinites, CustomJSONEncoder, @@ -52,8 +53,7 @@ def json_renderer(request, args, data, error, truncated=None): if error: shape = "objects" status_code = 400 - data["error"] = error - data["ok"] = False + data.update(error_body(error, status_code)) if truncated is not None: data["truncated"] = truncated @@ -87,7 +87,8 @@ def json_renderer(request, args, data, error, truncated=None): object_rows[pk_string] = row data = object_rows if shape_error: - data = {"ok": False, "error": shape_error} + status_code = 400 + data = error_body(shape_error, status_code) elif shape == "array": data = data["rows"] @@ -100,12 +101,7 @@ def json_renderer(request, args, data, error, truncated=None): data["rows"] = [list(row.values()) for row in data["rows"]] else: status_code = 400 - data = { - "ok": False, - "error": f"Invalid _shape: {shape}", - "status": 400, - "title": None, - } + data = error_body(f"Invalid _shape: {shape}", status_code) # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 diff --git a/datasette/static/app.css b/datasette/static/app.css index d6b37ff1..d101e4b7 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1518,183 +1518,6 @@ dialog.row-delete-dialog::backdrop { cursor: wait; } -dialog.table-label-columns-dialog { - --ink: #0f0f0f; - --paper: #eef6ff; - --muted: #6b6b6b; - --rule: #d8e6f5; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(440px, calc(100vw - 32px)); - max-width: 95vw; - max-height: min(560px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.table-label-columns-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.table-label-columns-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.table-label-columns-dialog .table-label-columns-form { - display: flex; - flex-direction: column; - min-height: 0; -} - -.table-label-columns-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-start; - gap: 12px; - flex-shrink: 0; - min-width: 0; -} - -.table-label-columns-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.table-label-columns-help, -.table-label-columns-error { - margin: 0; - padding: 16px 24px 0; -} - -.table-label-columns-help { - color: var(--muted); - font-size: 0.85rem; -} - -.table-label-columns-error { - color: #b91c1c; - font-size: 0.9rem; -} - -.table-label-columns-list { - list-style: none; - margin: 12px 0 0; - padding: 0 24px; - overflow-y: auto; -} - -.table-label-columns-row { - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - padding: 6px 0; - border-bottom: 1px solid var(--rule); -} - -.table-label-columns-label { - display: flex; - align-items: center; - gap: 8px; - min-width: 0; - color: var(--ink); - font-size: 0.9rem; -} - -.table-label-columns-name { - overflow-wrap: anywhere; - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; -} - -.table-label-columns-move-controls { - display: flex; - gap: 4px; - flex-shrink: 0; -} - -.table-label-columns-move-controls button { - border: 1px solid var(--rule); - background: var(--paper); - border-radius: 4px; - width: 26px; - height: 26px; - cursor: pointer; - color: var(--ink); -} - -.table-label-columns-move-controls button:disabled { - opacity: 0.4; - cursor: default; -} - -.table-label-columns-dialog .modal-footer { - padding: 18px 20px 14px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: flex-end; - gap: 10px; - flex-shrink: 0; - background: var(--paper); - margin-top: 18px; -} - -.table-label-columns-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.table-label-columns-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); - margin-right: auto; -} - -.table-label-columns-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.table-label-columns-dialog .btn-ghost.table-label-columns-cancel { - margin-right: 0; -} - -.table-label-columns-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.table-label-columns-dialog .btn-primary:hover { - background: #1949b8; -} - -.table-label-columns-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - dialog.row-edit-dialog { --ink: #0f0f0f; --paper: #eef6ff; @@ -1818,6 +1641,11 @@ dialog.row-edit-dialog::backdrop { overflow-y: auto; } +.row-edit-fields[hidden], +.row-edit-bulk[hidden] { + display: none; +} + .row-edit-field { display: grid; grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); @@ -1877,6 +1705,118 @@ textarea.row-edit-input { background: var(--paper); } +.row-edit-binary-control { + display: grid; + gap: 8px; + box-sizing: border-box; + width: 100%; + min-width: 0; + border: 1px solid var(--rule); + border-radius: 5px; + padding: 10px; + background: #fff; +} + +.row-edit-binary-control:focus { + border-color: var(--accent); + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.row-edit-binary-preview[hidden] { + display: none; +} + +.row-edit-binary-preview img { + display: block; + max-width: min(240px, 100%); + max-height: 180px; + border: 1px solid var(--rule); + border-radius: 4px; + background: var(--paper); +} + +.row-edit-binary-status { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 8px; + min-width: 0; +} + +.row-edit-binary-size { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.86rem; +} + +.row-edit-binary-name { + color: var(--muted); + font-size: 0.82rem; + overflow-wrap: anywhere; +} + +.row-edit-binary-name[hidden] { + display: none; +} + +.row-edit-binary-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.row-edit-binary-file-button, +.row-edit-binary-clear { + appearance: none; + border: 1px solid var(--rule); + border-radius: 4px; + background: #fff; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.78rem; + line-height: 1.2; + padding: 6px 8px; +} + +.row-edit-binary-file-button:hover, +.row-edit-binary-file-button:focus-within, +.row-edit-binary-clear:hover, +.row-edit-binary-clear:focus { + background: #f8fafc; +} + +.row-edit-binary-file-button:focus-within, +.row-edit-binary-clear:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.row-edit-binary-file-button input[type="file"] { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; + overflow: hidden; +} + +.row-edit-binary-clear[hidden] { + display: none; +} + +.row-edit-binary-drop-target { + border: 1px dashed var(--rule); + border-radius: 4px; + padding: 7px 8px; + color: var(--muted); + font-size: 0.78rem; +} + +.row-edit-binary-dragover .row-edit-binary-drop-target { + border-color: var(--accent); + background: var(--paper); + color: var(--ink); +} + .row-edit-default { display: grid; grid-template-columns: minmax(0, 1fr) 7.25rem; @@ -1975,6 +1915,207 @@ textarea.row-edit-input { margin: 0; } +.row-edit-bulk { + display: grid; + gap: 8px; + padding: 16px 24px 24px; + overflow-y: auto; +} + +.row-edit-bulk-editor { + display: grid; + gap: 8px; +} + +.row-edit-bulk-editor[hidden] { + display: none; +} + +.row-edit-bulk-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px; + justify-content: flex-start; +} + +.row-edit-bulk-actions .btn { + padding-left: 12px; + padding-right: 12px; +} + +.row-edit-bulk-conflict { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 8px 12px; + align-items: start; +} + +.row-edit-bulk-conflict[hidden] { + display: none; +} + +.row-edit-bulk-conflict-label { + color: var(--ink); + font-size: 0.82rem; + padding-top: 8px; +} + +.row-edit-bulk-conflict-control { + display: grid; + gap: 4px; +} + +.row-edit-bulk-conflict-help { + color: var(--muted); + font-size: 0.78rem; + margin: 0; +} + +.row-edit-copy-template-label-narrow { + display: none; +} + +.row-edit-bulk-template-note { + color: var(--muted); + font-size: 0.82rem; +} + +.row-edit-bulk-template-note-narrow { + display: none; +} + +.row-edit-bulk-textarea { + min-height: 16rem; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.45; +} + +.row-edit-bulk-textarea.row-edit-bulk-drop-target { + border-color: var(--accent); + background: #f8fbff; + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.row-edit-bulk-note { + color: var(--muted); + font-size: 0.82rem; + margin: 0; +} + +.row-edit-bulk-note label, +.row-edit-bulk-note .button-as-link { + font: inherit; +} + +@media (max-width: 640px) { + .row-edit-copy-template-label-wide { + display: none; + } + + .row-edit-copy-template-label-narrow { + display: inline; + } + + .row-edit-bulk-template-note-wide { + display: none; + } + + .row-edit-bulk-template-note-narrow { + display: inline; + } +} + +.row-edit-bulk-preview { + display: grid; + gap: 8px; + margin-top: 8px; +} + +.row-edit-bulk-preview[hidden] { + display: none; +} + +.row-edit-bulk-preview-summary { + color: var(--ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0; +} + +.row-edit-bulk-preview-table-wrap { + border: 1px solid var(--rule); + border-radius: 5px; + max-height: 18rem; + overflow: auto; + background: #fff; +} + +.row-edit-bulk-preview-table { + border-collapse: collapse; + font-size: 0.78rem; + min-width: 100%; + width: max-content; +} + +.row-edit-bulk-preview-table th, +.row-edit-bulk-preview-table td { + border-bottom: 1px solid var(--rule); + border-right: 1px solid var(--rule); + max-width: 18rem; + overflow-wrap: anywhere; + padding: 6px 8px; + text-align: left; + vertical-align: top; + white-space: normal; +} + +.row-edit-bulk-preview-table th { + background: var(--paper); + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} + +.row-edit-bulk-preview-table tr:last-child td { + border-bottom: none; +} + +.row-edit-bulk-preview-table th:last-child, +.row-edit-bulk-preview-table td:last-child { + border-right: none; +} + +.row-edit-bulk-preview-null, +.row-edit-bulk-preview-auto { + color: var(--muted); + font-style: italic; +} + +.row-edit-bulk-progress { + display: grid; + gap: 6px; +} + +.row-edit-bulk-progress[hidden] { + display: none; +} + +.row-edit-bulk-progress-bar { + width: 100%; +} + +.row-edit-bulk-progress-status { + color: var(--ink); + font-size: 0.9rem; + margin: 0; +} + datasette-autocomplete { display: block; position: relative; @@ -2053,6 +2194,16 @@ datasette-autocomplete input[type="text"], background: var(--paper); } +.row-edit-mode-link { + color: var(--accent); + font-size: 0.9rem; + margin-right: auto; +} + +.row-edit-mode-link[hidden] { + display: none; +} + .row-edit-dialog .btn { border: none; border-radius: 5px; @@ -2249,6 +2400,120 @@ select.table-create-input { gap: 10px; } +.table-create-columns[hidden], +.table-create-data[hidden], +.table-create-data-editor[hidden], +.table-create-data-preview[hidden] { + display: none; +} + +.table-create-data, +.table-create-data-editor { + display: grid; + gap: 8px; +} + +.table-create-data-label { + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; +} + +.table-create-data-textarea { + min-height: 16rem; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.45; +} + +.table-create-data-textarea.table-create-data-drop-target { + border-color: var(--accent); + background: #f8fbff; + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.table-create-data-note { + color: var(--muted); + font-size: 0.82rem; + margin: 0; +} + +.table-create-data-note label, +.table-create-data-note .button-as-link { + font: inherit; +} + +.table-create-data-preview { + display: grid; + gap: 10px; +} + +.table-create-data-preview-summary { + color: var(--ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0; +} + +.table-create-data-pk-field { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 12px; + align-items: center; +} + +.table-create-data-preview-table-wrap { + border: 1px solid var(--rule); + border-radius: 5px; + max-height: 18rem; + overflow: auto; + background: #fff; +} + +.table-create-data-preview-table { + border-collapse: collapse; + font-size: 0.78rem; + min-width: 100%; + width: max-content; +} + +.table-create-data-preview-table th, +.table-create-data-preview-table td { + border-bottom: 1px solid var(--rule); + border-right: 1px solid var(--rule); + max-width: 18rem; + overflow-wrap: anywhere; + padding: 6px 8px; + text-align: left; + vertical-align: top; + white-space: normal; +} + +.table-create-data-preview-table th { + background: var(--paper); + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} + +.table-create-data-preview-table tr:last-child td { + border-bottom: none; +} + +.table-create-data-preview-table th:last-child, +.table-create-data-preview-table td:last-child { + border-right: none; +} + +.table-create-data-preview-null { + color: var(--muted); + font-style: italic; +} + .table-create-column-list { display: grid; gap: 8px; @@ -2476,6 +2741,16 @@ select.table-create-input { background: var(--paper); } +.table-create-mode-link { + color: var(--accent); + font-size: 0.9rem; + margin-right: auto; +} + +.table-create-mode-link[hidden] { + display: none; +} + .table-create-dialog .btn { border: none; border-radius: 5px; @@ -3189,7 +3464,8 @@ select.table-alter-input { .row-edit-dialog .modal-header, .row-edit-summary, .row-edit-loading, - .row-edit-fields { + .row-edit-fields, + .row-edit-bulk { padding-left: 18px; padding-right: 18px; } @@ -3208,6 +3484,15 @@ select.table-alter-input { padding-top: 0; } + .row-edit-bulk-conflict { + grid-template-columns: 1fr; + gap: 5px; + } + + .row-edit-bulk-conflict-label { + padding-top: 0; + } + .row-edit-dialog .modal-footer { padding-left: 18px; padding-right: 18px; @@ -3239,6 +3524,11 @@ select.table-alter-input { padding-top: 0; } + .table-create-data-pk-field { + grid-template-columns: 1fr; + gap: 5px; + } + .table-create-column-headings { display: none; } diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 7dd9acc2..9e8b93f6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -2,12 +2,12 @@ var ROW_DELETE_DIALOG_ID = "row-delete-dialog"; var rowDeleteDialogState = null; var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; +var ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES = 10 * 1024 * 1024; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; +var TABLE_CREATE_AUTOMATIC_PK = "__datasette_automatic_pk__"; var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; var tableAlterDialogState = null; -var TABLE_LABEL_COLUMNS_DIALOG_ID = "table-label-columns-dialog"; -var tableLabelColumnsDialogState = null; function ensureRowMutationStatus(manager) { var status = document.querySelector(".row-mutation-status"); @@ -812,12 +812,60 @@ async function loadTableCreateForeignKeyTargets(state) { ); } +function tableCreateIsDataMode(state) { + return state && state.mode === "data"; +} + +function tableCreateSaveButtonText(state) { + if (tableCreateIsDataMode(state)) { + return state.dataPreviewReady ? "Create table" : "Preview rows"; + } + return "Create table"; +} + +function tableCreateCanInsertRows() { + var data = databaseCreateTableData() || {}; + return !!data.canInsertRows; +} + +function syncTableCreateModeUi(state) { + if (!state) { + return; + } + var isDataMode = tableCreateIsDataMode(state); + state.columnsPanel.hidden = isDataMode; + state.dataPanel.hidden = !isDataMode; + state.dataEditor.hidden = !isDataMode || state.dataPreviewReady; + state.dataPreview.hidden = !isDataMode || !state.dataPreviewReady; + state.createFromDataLink.hidden = isDataMode || !tableCreateCanInsertRows(); + state.manualCreateLink.hidden = !isDataMode; +} + +function updateTableCreateDialogButtons(state) { + if (!state) { + return; + } + syncTableCreateModeUi(state); + state.cancelButton.disabled = state.isSaving; + state.saveButton.disabled = state.isSaving; + state.addColumnButton.disabled = state.isSaving; + state.cancelButton.textContent = + tableCreateIsDataMode(state) && state.dataPreviewReady ? "Back" : "Cancel"; + state.saveButton.textContent = state.isSaving + ? "Creating..." + : tableCreateSaveButtonText(state); +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; } - return JSON.stringify({ + var signature = { table: state.tableName.value, + data: state.dataTextarea ? state.dataTextarea.value : "", + dataPrimaryKey: state.dataPkSelect + ? state.dataPkSelect.value + : TABLE_CREATE_AUTOMATIC_PK, columns: tableCreateDialogRows(state).map(function (row) { return { name: row.querySelector(".table-create-column-name").value, @@ -840,7 +888,8 @@ function tableCreateDialogSignature(state) { ).value || "", }; }), - }); + }; + return JSON.stringify(signature); } function tableCreateDialogHasChanges(state) { @@ -866,18 +915,23 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.cancelButton.disabled = isSaving; - state.saveButton.disabled = isSaving; - state.addColumnButton.disabled = isSaving; - state.saveButton.textContent = isSaving ? "Creating..." : "Create table"; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { control.disabled = isSaving; }); + state.fields + .querySelectorAll( + ".table-create-data input, .table-create-data select, .table-create-data textarea, .table-create-data button", + ) + .forEach(function (control) { + control.disabled = isSaving; + }); + state.tableName.disabled = isSaving; if (!isSaving) { updateTableCreateColumnRules(state); } + updateTableCreateDialogButtons(state); updateTableCreateMoveButtons(state); } @@ -1233,8 +1287,11 @@ function addTableCreateColumn(state, column) { } function resetTableCreateDialog(state) { + state.mode = "manual"; state.nextColumnIndex = 0; state.tableName.value = ""; + state.dataTextarea.value = ""; + resetTableCreateDataPreview(state); state.columnList.textContent = ""; addTableCreateColumn(state, { name: "id", @@ -1247,9 +1304,39 @@ function resetTableCreateDialog(state) { primaryKey: false, }); updateTableCreateColumnRules(state); + updateTableCreateDialogButtons(state); state.initialSignature = tableCreateDialogSignature(state); } +function showTableCreateDataMode(state) { + if (!state || state.isSaving || !tableCreateCanInsertRows()) { + return; + } + state.mode = "data"; + clearTableCreateDialogError(state); + updateTableCreateDialogButtons(state); + if (state.dataPreviewReady && state.dataPkSelect) { + state.dataPkSelect.focus(); + } else { + state.dataTextarea.focus(); + } +} + +function showTableCreateManualMode(state) { + if (!state || state.isSaving) { + return; + } + state.mode = "manual"; + clearTableCreateDialogError(state); + updateTableCreateDialogButtons(state); + var firstInput = state.columnList.querySelector(".table-create-column-name"); + if (firstInput) { + firstInput.focus(); + } else { + state.tableName.focus(); + } +} + function collectTableCreatePayload(state) { var payload = { table: state.tableName.value.trim(), @@ -1317,14 +1404,9 @@ function collectTableCreateColumnTypeAssignments(state) { } function validateTableCreatePayload(payload) { - if (!payload.table) { - return "Table name is required."; - } - if (payload.table.indexOf("\n") !== -1) { - return "Table name cannot contain newlines."; - } - if (/^sqlite_/i.test(payload.table)) { - return "Table name cannot start with sqlite_."; + var tableNameError = validateTableCreateTableName(payload.table); + if (tableNameError) { + return tableNameError; } if (!payload.columns.length) { return "At least one column is required."; @@ -1354,6 +1436,19 @@ function validateTableCreatePayload(payload) { return null; } +function validateTableCreateTableName(tableName) { + if (!tableName) { + return "Table name is required."; + } + if (tableName.indexOf("\n") !== -1) { + return "Table name cannot contain newlines."; + } + if (/^sqlite_/i.test(tableName)) { + return "Table name cannot start with sqlite_."; + } + return null; +} + function validateTableCreateColumnTypeAssignments(assignments) { for (var i = 0; i < assignments.length; i += 1) { var assignment = assignments[i]; @@ -1376,6 +1471,476 @@ function validateTableCreateColumnTypeAssignments(assignments) { return null; } +function normalizeCreateTableDataJsonValue(value) { + if (typeof value === "undefined") { + return null; + } + if (Array.isArray(value) || (value && typeof value === "object")) { + return JSON.stringify(value); + } + return value; +} + +function createTableDataColumnObjects(names) { + return names.map(function (name) { + return { name: name }; + }); +} + +function validateCreateTableDataHeaders(headers) { + if (!headers.length) { + throw new Error("No columns found to preview."); + } + var seen = {}; + headers.forEach(function (name, index) { + if (!name) { + throw new Error("Column header " + (index + 1) + " is blank."); + } + if (name.indexOf("\n") !== -1) { + throw new Error("Column names cannot contain newlines."); + } + var key = name.toLowerCase(); + if (seen[key]) { + throw new Error("Duplicate column name: " + name); + } + seen[key] = true; + }); +} + +function jsonRowIsObject(item) { + return !!(item && typeof item === "object" && !Array.isArray(item)); +} + +function extractJsonObjectRows(parsed) { + if (Array.isArray(parsed)) { + return parsed; + } + if (!jsonRowIsObject(parsed)) { + throw new Error( + "JSON must be an array of objects, or an object containing an array of objects.", + ); + } + + var bestRows = null; + Object.keys(parsed).forEach(function (key) { + var value = parsed[key]; + if (!Array.isArray(value) || !value.every(jsonRowIsObject)) { + return; + } + if (!bestRows || value.length > bestRows.length) { + bestRows = value; + } + }); + if (!bestRows) { + throw new Error( + "JSON object must contain at least one root key with an array of objects.", + ); + } + return bestRows; +} + +function parseJsonObjectRows(text) { + var parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new Error("Invalid JSON: " + error.message); + } + var rows = extractJsonObjectRows(parsed); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + return rows; +} + +function parseJsonCreateTableRows(text) { + var parsed = parseJsonObjectRows(text); + + var columnNames = []; + var columnMap = {}; + parsed.forEach(function (item, index) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("JSON row " + (index + 1) + " must be an object."); + } + Object.keys(item).forEach(function (name) { + if (!columnMap[name]) { + columnMap[name] = true; + columnNames.push(name); + } + }); + }); + validateCreateTableDataHeaders(columnNames); + + var rows = parsed.map(function (item) { + var row = {}; + columnNames.forEach(function (name) { + row[name] = Object.prototype.hasOwnProperty.call(item, name) + ? normalizeCreateTableDataJsonValue(item[name]) + : null; + }); + return row; + }); + return { + columns: createTableDataColumnObjects(columnNames), + rows: rows, + }; +} + +function createTableDelimitedValueIsInteger(value) { + return /^[-+]?\d+$/.test(String(value).trim()); +} + +function createTableDelimitedValueIsFloat(value) { + var trimmed = String(value).trim(); + if (!trimmed) { + return false; + } + var numberValue = Number(trimmed); + return Number.isFinite(numberValue); +} + +function inferCreateTableDelimitedColumnType(values) { + var nonBlankValues = values.filter(function (value) { + return String(value).trim() !== ""; + }); + if (!nonBlankValues.length) { + return "text"; + } + if (nonBlankValues.every(createTableDelimitedValueIsInteger)) { + return "integer"; + } + if (nonBlankValues.every(createTableDelimitedValueIsFloat)) { + return "float"; + } + return "text"; +} + +function coerceCreateTableDelimitedValue(value, type) { + var trimmed = String(value).trim(); + if (trimmed === "") { + return type === "integer" || type === "float" ? null : ""; + } + if (type === "integer") { + return parseInt(trimmed, 10); + } + if (type === "float") { + return Number(trimmed); + } + return value; +} + +function detectCreateTableDataDelimiter(text) { + var firstLine = + text.split(/\r\n|\n|\r/).find(function (line) { + return line.trim() !== ""; + }) || ""; + var csvRows = delimiterPreviewRows(firstLine, ","); + var tsvRows = delimiterPreviewRows(firstLine, "\t"); + var csvColumns = csvRows.length ? csvRows[0].length : 0; + var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; + + if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { + return "\t"; + } + if (tsvColumns > csvColumns) { + return "\t"; + } + if (csvColumns > 1) { + return ","; + } + if (tsvColumns > 1) { + return "\t"; + } + return null; +} + +function parseDelimitedCreateTableRows(text) { + var delimiter = detectCreateTableDataDelimiter(text); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + + var headers = rows[0].map(function (value) { + return value.trim(); + }); + validateCreateTableDataHeaders(headers); + var dataRows = rows.slice(1); + if (!dataRows.length) { + throw new Error("No data rows found to preview."); + } + + dataRows.forEach(function (row, index) { + if (row.length > headers.length) { + throw new Error( + "Row " + + (index + 1) + + " has " + + row.length + + " values, but only " + + headers.length + + " columns were provided.", + ); + } + }); + + var columnTypes = headers.map(function (_name, columnIndex) { + return inferCreateTableDelimitedColumnType( + dataRows.map(function (row) { + return row[columnIndex] || ""; + }), + ); + }); + + return { + columns: createTableDataColumnObjects(headers), + rows: dataRows.map(function (row) { + var rowObject = {}; + headers.forEach(function (name, columnIndex) { + rowObject[name] = coerceCreateTableDelimitedValue( + row[columnIndex] || "", + columnTypes[columnIndex], + ); + }); + return rowObject; + }), + }; +} + +function parseCreateTableDataRows(text) { + var trimmed = text.trim(); + if (!trimmed) { + throw new Error("Paste rows before previewing."); + } + if (trimmed[0] === "[" || trimmed[0] === "{") { + return parseJsonCreateTableRows(trimmed); + } + return parseDelimitedCreateTableRows(trimmed); +} + +function tableCreateDataRecommendedPrimaryKey(columns, rows) { + var candidates = []; + columns.forEach(function (column, columnIndex) { + var distinctValues = {}; + var maxLength = 0; + var valid = rows.length > 0; + rows.forEach(function (row) { + if (!valid) { + return; + } + var value = row[column.name]; + if (value === null || typeof value === "undefined") { + valid = false; + return; + } + var text = String(value).trim(); + if (!text || text.length >= 20 || /\s/.test(text)) { + valid = false; + return; + } + if (distinctValues[text]) { + valid = false; + return; + } + distinctValues[text] = true; + maxLength = Math.max(maxLength, text.length); + }); + if (valid) { + candidates.push({ + name: column.name, + maxLength: maxLength, + columnIndex: columnIndex, + }); + } + }); + candidates.sort(function (left, right) { + if (left.maxLength !== right.maxLength) { + return left.maxLength - right.maxLength; + } + return left.columnIndex - right.columnIndex; + }); + return candidates.length ? candidates[0].name : TABLE_CREATE_AUTOMATIC_PK; +} + +function renderTableCreateDataPreview(state, preview) { + state.dataPreview.textContent = ""; + + var summary = document.createElement("p"); + summary.className = "table-create-data-preview-summary"; + summary.textContent = + "Previewing " + + preview.rows.length + + " row" + + (preview.rows.length === 1 ? "." : "s."); + state.dataPreview.appendChild(summary); + + var pkField = document.createElement("div"); + pkField.className = "table-create-data-pk-field"; + var pkLabel = document.createElement("label"); + pkLabel.className = "table-create-data-label"; + pkLabel.setAttribute("for", "table-create-data-primary-key"); + pkLabel.textContent = "Primary key"; + var pkSelect = document.createElement("select"); + pkSelect.id = "table-create-data-primary-key"; + pkSelect.className = "table-create-input table-create-data-primary-key"; + var automaticOption = document.createElement("option"); + automaticOption.value = TABLE_CREATE_AUTOMATIC_PK; + automaticOption.textContent = "Automatic ID column"; + pkSelect.appendChild(automaticOption); + preview.columns.forEach(function (column) { + var option = document.createElement("option"); + option.value = column.name; + option.textContent = column.name; + pkSelect.appendChild(option); + }); + pkSelect.value = tableCreateDataRecommendedPrimaryKey( + preview.columns, + preview.rows, + ); + pkSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + }); + state.dataPkSelect = pkSelect; + pkField.appendChild(pkLabel); + pkField.appendChild(pkSelect); + state.dataPreview.appendChild(pkField); + + var tableWrap = document.createElement("div"); + tableWrap.className = "table-create-data-preview-table-wrap"; + var table = document.createElement("table"); + table.className = "table-create-data-preview-table"; + var thead = document.createElement("thead"); + var headerRow = document.createElement("tr"); + preview.columns.forEach(function (column) { + var th = document.createElement("th"); + th.scope = "col"; + th.textContent = column.name; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + var tbody = document.createElement("tbody"); + preview.rows.forEach(function (row) { + var tr = document.createElement("tr"); + preview.columns.forEach(function (column) { + var td = document.createElement("td"); + var value = row[column.name]; + td.textContent = bulkInsertPreviewValue(value); + if (value === null) { + td.className = "table-create-data-preview-null"; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + tableWrap.appendChild(table); + state.dataPreview.appendChild(tableWrap); + state.dataPreview.hidden = false; +} + +function resetTableCreateDataPreview(state) { + state.dataPreviewRows = null; + state.dataPreviewColumns = []; + state.dataPreviewReady = false; + state.dataPkSelect = null; + state.dataPreview.hidden = true; + state.dataPreview.textContent = ""; + syncTableCreateModeUi(state); +} + +function tableCreateTableNameFromFileName(fileName) { + var baseName = (fileName || "").replace(/^.*[\\/]/, ""); + var nameWithoutExtension = baseName.replace(/\.[^.]*$/, ""); + return nameWithoutExtension + .trim() + .replace(/\s+/g, "_") + .toLowerCase() + .replace(/[^a-z0-9_]/g, ""); +} + +async function loadTableCreateDataTextFile(state, file) { + if (!file) { + return; + } + try { + var text = await readTextFile(file); + var tableName = tableCreateTableNameFromFileName(file.name); + if (tableName) { + state.tableName.value = tableName; + state.tableName.dispatchEvent(new Event("input", { bubbles: true })); + } + state.dataTextarea.value = text; + state.dataTextarea.dispatchEvent(new Event("input", { bubbles: true })); + clearTableCreateDialogError(state); + state.dataTextarea.focus(); + } catch (_error) { + showTableCreateDialogError(state, "Could not read that text file."); + } +} + +function previewTableCreateDataRows(state) { + clearTableCreateDialogError(state); + resetTableCreateDataPreview(state); + try { + var preview = parseCreateTableDataRows(state.dataTextarea.value); + state.dataPreviewRows = preview.rows; + state.dataPreviewColumns = preview.columns; + state.dataPreviewReady = true; + renderTableCreateDataPreview(state, preview); + updateTableCreateDialogButtons(state); + } catch (error) { + showTableCreateDialogError( + state, + error.message || "Could not preview rows.", + ); + updateTableCreateDialogButtons(state); + } +} + +function collectTableCreateDataPayload(state) { + var payload = { + table: state.tableName.value.trim(), + rows: state.dataPreviewRows || [], + }; + var primaryKey = state.dataPkSelect + ? state.dataPkSelect.value + : TABLE_CREATE_AUTOMATIC_PK; + payload.pk = + primaryKey === TABLE_CREATE_AUTOMATIC_PK ? "id" : primaryKey || undefined; + return payload; +} + +function validateTableCreateDataPayload(payload, state) { + var tableNameError = validateTableCreateTableName(payload.table); + if (tableNameError) { + return tableNameError; + } + if (!payload.rows.length) { + return "No rows found to create."; + } + if ( + state.dataPkSelect && + state.dataPkSelect.value === TABLE_CREATE_AUTOMATIC_PK && + payload.rows.some(function (row) { + return Object.prototype.hasOwnProperty.call(row, "id"); + }) + ) { + return ( + "Automatic ID column cannot be used because the pasted data " + + "already has an id column." + ); + } + return null; +} + function fallbackTableUrl(tableName) { var data = databaseCreateTableData() || {}; if (!data.path) { @@ -1443,6 +2008,57 @@ async function assignTableCreateColumnTypes( } } +async function createTableFromDataPreview(state) { + var data = databaseCreateTableData(); + if (!data || !data.path) { + showTableCreateDialogError(state, "Could not find the create table URL."); + return; + } + var payload = collectTableCreateDataPayload(state); + var validationError = validateTableCreateDataPayload(payload, state); + if (validationError) { + showTableCreateDialogError(state, validationError); + return; + } + clearTableCreateDialogError(state); + setTableCreateDialogSaving(state, true); + try { + var response = await fetch(data.path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + var tableUrl = + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); + state.shouldRestoreFocus = false; + state.dialog.close(); + if (tableUrl) { + location.href = tableUrl; + } else { + location.reload(); + } + } catch (error) { + setTableCreateDialogSaving(state, false); + showTableCreateDialogError( + state, + error.message || "Could not create table", + ); + } +} + async function saveTableCreateDialog(state) { if (state.isSaving) { return; @@ -1452,6 +2068,14 @@ async function saveTableCreateDialog(state) { showTableCreateDialogError(state, "Could not find the create table URL."); return; } + if (tableCreateIsDataMode(state)) { + if (!state.dataPreviewReady) { + previewTableCreateDataRows(state); + } else { + await createTableFromDataPreview(state); + } + return; + } clearTableCreateDialogError(state); var payload = collectTableCreatePayload(state); var columnTypeAssignments = collectTableCreateColumnTypeAssignments(state); @@ -1562,8 +2186,18 @@ function ensureTableCreateDialog(manager) {
+
+ Create table from data +
@@ -1578,13 +2212,27 @@ function ensureTableCreateDialog(manager) { error: dialog.querySelector(".table-create-error"), fields: dialog.querySelector(".table-create-fields"), tableName: dialog.querySelector(".table-create-table-name"), + columnsPanel: dialog.querySelector(".table-create-columns"), columnList: dialog.querySelector(".table-create-column-list"), addColumnButton: dialog.querySelector(".table-create-add-column"), + dataPanel: dialog.querySelector(".table-create-data"), + dataEditor: dialog.querySelector(".table-create-data-editor"), + dataTextarea: dialog.querySelector(".table-create-data-textarea"), + dataOpenFileButton: dialog.querySelector(".table-create-data-open-file"), + dataFileInput: dialog.querySelector(".table-create-data-file-input"), + dataPreview: dialog.querySelector(".table-create-data-preview"), + createFromDataLink: dialog.querySelector(".table-create-from-data"), + manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), currentButton: null, shouldRestoreFocus: true, isSaving: false, + mode: "manual", + dataPreviewRows: null, + dataPreviewColumns: [], + dataPreviewReady: false, + dataPkSelect: null, initialSignature: "", nextColumnIndex: 0, foreignKeyTargets: [], @@ -1608,13 +2256,114 @@ function ensureTableCreateDialog(manager) { }); tableCreateDialogState.cancelButton.addEventListener("click", function () { + if ( + tableCreateIsDataMode(tableCreateDialogState) && + tableCreateDialogState.dataPreviewReady && + !tableCreateDialogState.isSaving + ) { + resetTableCreateDataPreview(tableCreateDialogState); + updateTableCreateDialogButtons(tableCreateDialogState); + tableCreateDialogState.dataTextarea.focus(); + return; + } closeTableCreateDialogIfConfirmed(tableCreateDialogState); }); + tableCreateDialogState.createFromDataLink.addEventListener( + "click", + function (ev) { + ev.preventDefault(); + showTableCreateDataMode(tableCreateDialogState); + }, + ); + + tableCreateDialogState.manualCreateLink.addEventListener( + "click", + function (ev) { + ev.preventDefault(); + showTableCreateManualMode(tableCreateDialogState); + }, + ); + tableCreateDialogState.tableName.addEventListener("input", function () { clearTableCreateDialogError(tableCreateDialogState); }); + tableCreateDialogState.dataOpenFileButton.addEventListener( + "click", + function () { + tableCreateDialogState.dataFileInput.click(); + }, + ); + + tableCreateDialogState.dataFileInput.addEventListener( + "change", + async function (ev) { + var files = ev.target.files; + await loadTableCreateDataTextFile( + tableCreateDialogState, + files && files.length ? files[0] : null, + ); + ev.target.value = ""; + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragenter", + function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.add( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragover", + function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.add( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragleave", + function () { + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "drop", + async function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + var files = ev.dataTransfer && ev.dataTransfer.files; + if (!files || !files.length) { + return; + } + await loadTableCreateDataTextFile(tableCreateDialogState, files[0]); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener("dragend", function () { + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + }); + + tableCreateDialogState.dataTextarea.addEventListener("input", function () { + resetTableCreateDataPreview(tableCreateDialogState); + clearTableCreateDialogError(tableCreateDialogState); + updateTableCreateDialogButtons(tableCreateDialogState); + }); + dialog.addEventListener("click", function (ev) { if (ev.target === dialog) { closeTableCreateDialogIfConfirmed(tableCreateDialogState); @@ -1799,10 +2548,6 @@ function tableAlterData() { return tablePageData().alterTable; } -function tableLabelColumnsData() { - return tablePageData().labelColumns; -} - function tableAlterColumnTypes() { var data = tableAlterData() || {}; return data.columnTypes && data.columnTypes.length @@ -3588,6 +4333,14 @@ function tableInsertUrl() { return url.toString(); } +function tableUpsertUrl() { + var data = tableInsertData(); + if (data && data.upsertPath) { + return new URL(data.upsertPath, location.href).toString(); + } + return null; +} + function rowResourceUrl(row) { var rowId = row.getAttribute("data-row"); if (!rowId) { @@ -3604,7 +4357,7 @@ function rowJsonUrl(row) { return ""; } url.pathname = url.pathname + ".json"; - url.searchParams.set("_extra", "columns,column_types"); + url.searchParams.set("_extra", "columns,column_types,column_details"); return url.toString(); } @@ -3884,10 +4637,155 @@ function initRowDeleteActions(manager) { }); } +function isBase64JsonValue(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + var keys = Object.keys(value); + return ( + keys.length === 2 && + Object.prototype.hasOwnProperty.call(value, "$base64") && + Object.prototype.hasOwnProperty.call(value, "encoded") && + value.$base64 === true && + typeof value.encoded === "string" + ); +} + +function shouldUseBinaryControl(value, options) { + options = options || {}; + var sqliteType = (options.sqliteType || "").toLowerCase(); + return ( + isBase64JsonValue(value) || + sqliteType === "blob" || + options.valueKind === "binary" + ); +} + +function binaryEncodedValue(value) { + return isBase64JsonValue(value) ? value.encoded : ""; +} + +function binaryByteLengthFromBase64(encoded) { + encoded = (encoded || "").replace(/\s/g, ""); + if (!encoded) { + return 0; + } + var padding = 0; + if (encoded.slice(-2) === "==") { + padding = 2; + } else if (encoded.slice(-1) === "=") { + padding = 1; + } + return Math.max(0, Math.floor((encoded.length * 3) / 4) - padding); +} + +function rowEditBinaryValue(encoded) { + return { + $base64: true, + encoded: encoded || "", + }; +} + +function formatRowEditBinarySize(byteLength) { + var number = byteLength.toLocaleString(); + return "Binary: " + number + " byte" + (byteLength === 1 ? "" : "s"); +} + +function base64ToUint8Array(encoded) { + var binary = window.atob(encoded || ""); + var bytes = new Uint8Array(binary.length); + for (var i = 0; i < binary.length; i += 1) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +function uint8ArrayToBase64(bytes) { + var chunks = []; + var chunkSize = 0x8000; + for (var i = 0; i < bytes.length; i += chunkSize) { + chunks.push( + String.fromCharCode.apply(null, bytes.subarray(i, i + chunkSize)), + ); + } + return window.btoa(chunks.join("")); +} + +function rowEditBinaryImageMimeType(bytes) { + if (!bytes || !bytes.length) { + return null; + } + if ( + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 && + bytes[4] === 0x0d && + bytes[5] === 0x0a && + bytes[6] === 0x1a && + bytes[7] === 0x0a + ) { + return "image/png"; + } + if ( + bytes.length >= 3 && + bytes[0] === 0xff && + bytes[1] === 0xd8 && + bytes[2] === 0xff + ) { + return "image/jpeg"; + } + if ( + bytes.length >= 6 && + bytes[0] === 0x47 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x38 && + (bytes[4] === 0x37 || bytes[4] === 0x39) && + bytes[5] === 0x61 + ) { + return "image/gif"; + } + if ( + bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ) { + return "image/webp"; + } + if ( + bytes.length >= 12 && + bytes[4] === 0x66 && + bytes[5] === 0x74 && + bytes[6] === 0x79 && + bytes[7] === 0x70 && + bytes[8] === 0x61 && + bytes[9] === 0x76 && + bytes[10] === 0x69 && + (bytes[11] === 0x66 || bytes[11] === 0x73) + ) { + return "image/avif"; + } + if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) { + return "image/bmp"; + } + return null; +} + function valueToEditText(value) { if (value === null || typeof value === "undefined") { return ""; } + if (isBase64JsonValue(value)) { + return value.encoded; + } if (typeof value === "object") { return JSON.stringify(value, null, 2); } @@ -3898,6 +4796,9 @@ function shouldUseTextarea(value, columnType) { if (columnType && columnType.type === "textarea") { return true; } + if (isBase64JsonValue(value)) { + return false; + } if (value && typeof value === "object") { return true; } @@ -3906,6 +4807,9 @@ function shouldUseTextarea(value, columnType) { } function rowEditValueKind(value) { + if (isBase64JsonValue(value)) { + return "binary"; + } if (value === null || typeof value === "undefined") { return "null"; } @@ -3929,6 +4833,261 @@ function rowEditControlElement(control, autocompleteUrl) { return autocomplete; } +function revokeRowEditBinaryPreview(wrapper) { + if (wrapper && wrapper._rowEditBinaryPreviewUrl) { + URL.revokeObjectURL(wrapper._rowEditBinaryPreviewUrl); + wrapper._rowEditBinaryPreviewUrl = null; + } +} + +function updateRowEditBinaryPreview(wrapper, encoded, byteLength) { + var preview = wrapper.querySelector(".row-edit-binary-preview"); + if (!preview) { + return; + } + revokeRowEditBinaryPreview(wrapper); + preview.hidden = true; + preview.textContent = ""; + if ( + !encoded || + byteLength >= ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES || + !window.atob || + !window.Blob || + !window.URL || + !URL.createObjectURL + ) { + return; + } + + var bytes; + try { + bytes = base64ToUint8Array(encoded); + } catch (_error) { + return; + } + var mimeType = rowEditBinaryImageMimeType(bytes); + if (!mimeType) { + return; + } + + var image = document.createElement("img"); + image.alt = ""; + var objectUrl = URL.createObjectURL(new Blob([bytes], { type: mimeType })); + wrapper._rowEditBinaryPreviewUrl = objectUrl; + + var showPreview = function () { + if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { + preview.hidden = false; + } + }; + var hidePreview = function () { + if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { + revokeRowEditBinaryPreview(wrapper); + preview.hidden = true; + preview.textContent = ""; + } + }; + image.onload = showPreview; + image.onerror = hidePreview; + image.src = objectUrl; + preview.appendChild(image); +} + +function updateRowEditBinaryDisplay(wrapper, control, fileName) { + var kind = rowEditControlValueKind(control); + var size = wrapper.querySelector(".row-edit-binary-size"); + var name = wrapper.querySelector(".row-edit-binary-name"); + var clear = wrapper.querySelector(".row-edit-binary-clear"); + var byteLength = + kind === "binary" ? binaryByteLengthFromBase64(control.value) : null; + + if (size) { + size.textContent = + kind === "binary" + ? formatRowEditBinarySize(byteLength) + : "No binary data"; + } + if (name) { + name.textContent = fileName || ""; + name.hidden = !fileName; + } + if (clear) { + clear.hidden = control.dataset.notNull === "1" || kind !== "binary"; + } + if (kind === "binary") { + updateRowEditBinaryPreview(wrapper, control.value, byteLength); + } else { + updateRowEditBinaryPreview(wrapper, "", 0); + } +} + +function setRowEditBinaryControlValue(control, encoded, fileName) { + control.value = encoded || ""; + control.dataset.currentValueKind = "binary"; + updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, fileName); + control.dispatchEvent(new Event("input", { bubbles: true })); +} + +function clearRowEditBinaryControlValue(control) { + control.value = ""; + control.dataset.currentValueKind = "null"; + updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, ""); + control.dispatchEvent(new Event("input", { bubbles: true })); +} + +function readRowEditBinaryFile(file) { + return new Promise(function (resolve, reject) { + var reader = new FileReader(); + reader.onload = function () { + try { + var bytes = new Uint8Array(reader.result || new ArrayBuffer(0)); + resolve({ + encoded: uint8ArrayToBase64(bytes), + name: file && file.name ? file.name : "", + }); + } catch (error) { + reject(error); + } + }; + reader.onerror = function () { + reject(reader.error || new Error("Could not read file")); + }; + reader.readAsArrayBuffer(file); + }); +} + +function rowEditBinaryValueFromText(text) { + var encoder = new TextEncoder(); + var bytes = encoder.encode(text || ""); + return { + encoded: uint8ArrayToBase64(bytes), + name: "Pasted text", + }; +} + +function rowEditBinaryFirstClipboardFile(clipboardData) { + if (!clipboardData) { + return null; + } + if (clipboardData.files && clipboardData.files.length) { + return clipboardData.files[0]; + } + var items = clipboardData.items || []; + for (var i = 0; i < items.length; i += 1) { + if (items[i].kind === "file") { + return items[i].getAsFile(); + } + } + return null; +} + +function handleRowEditBinaryFile(control, file) { + if (!file) { + return; + } + readRowEditBinaryFile(file) + .then(function (value) { + setRowEditBinaryControlValue(control, value.encoded, value.name); + }) + .catch(function (error) { + console.error("Could not read binary file", error); + }); +} + +function createRowEditBinaryControlElement(control, value, options, labelId) { + var wrapper = document.createElement("div"); + wrapper.className = "row-edit-binary-control"; + wrapper.dataset.column = control.name; + wrapper.setAttribute("role", "group"); + wrapper.setAttribute("tabindex", "0"); + wrapper.setAttribute("aria-labelledby", labelId); + + var preview = document.createElement("div"); + preview.className = "row-edit-binary-preview"; + preview.hidden = true; + + var status = document.createElement("div"); + status.className = "row-edit-binary-status"; + var size = document.createElement("span"); + size.className = "row-edit-binary-size"; + var name = document.createElement("span"); + name.className = "row-edit-binary-name"; + name.hidden = true; + status.appendChild(size); + status.appendChild(name); + + var actions = document.createElement("div"); + actions.className = "row-edit-binary-actions"; + var fileLabel = document.createElement("label"); + fileLabel.className = "row-edit-binary-file-button"; + fileLabel.textContent = "Attach file"; + var fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.setAttribute("aria-label", "Attach file for " + control.name); + fileInput.addEventListener("change", function () { + if (fileInput.files && fileInput.files.length) { + handleRowEditBinaryFile(control, fileInput.files[0]); + } + }); + fileLabel.appendChild(fileInput); + actions.appendChild(fileLabel); + + var clearButton = document.createElement("button"); + clearButton.type = "button"; + clearButton.className = "row-edit-binary-clear"; + clearButton.textContent = "Set NULL"; + clearButton.addEventListener("click", function () { + clearRowEditBinaryControlValue(control); + wrapper.focus(); + }); + actions.appendChild(clearButton); + + var dropTarget = document.createElement("div"); + dropTarget.className = "row-edit-binary-drop-target"; + dropTarget.textContent = "Drop or paste file contents"; + ["dragenter", "dragover"].forEach(function (eventName) { + wrapper.addEventListener(eventName, function (ev) { + ev.preventDefault(); + wrapper.classList.add("row-edit-binary-dragover"); + }); + }); + ["dragleave", "drop"].forEach(function (eventName) { + wrapper.addEventListener(eventName, function () { + wrapper.classList.remove("row-edit-binary-dragover"); + }); + }); + wrapper.addEventListener("drop", function (ev) { + ev.preventDefault(); + var file = ev.dataTransfer && ev.dataTransfer.files[0]; + handleRowEditBinaryFile(control, file); + }); + wrapper.addEventListener("paste", function (ev) { + var file = rowEditBinaryFirstClipboardFile(ev.clipboardData); + if (file) { + ev.preventDefault(); + handleRowEditBinaryFile(control, file); + return; + } + var text = ev.clipboardData && ev.clipboardData.getData("text"); + if (text) { + ev.preventDefault(); + var pasted = rowEditBinaryValueFromText(text); + setRowEditBinaryControlValue(control, pasted.encoded, pasted.name); + } + }); + + control.type = "hidden"; + control._rowEditBinaryWrapper = wrapper; + wrapper.appendChild(control); + wrapper.appendChild(preview); + wrapper.appendChild(status); + wrapper.appendChild(actions); + wrapper.appendChild(dropTarget); + + updateRowEditBinaryDisplay(wrapper, control, ""); + return wrapper; +} + function columnTypeForContext(columnType) { if (!columnType) { return null; @@ -4192,6 +5351,11 @@ function focusFirstRowEditControl(state, options) { if (focusRowEditPluginControl(field)) { return true; } + var binaryControl = field.querySelector(".row-edit-binary-control"); + if (binaryControl) { + binaryControl.focus(); + return true; + } control.focus(); return true; } @@ -4215,6 +5379,11 @@ function destroyRowEditFields(state) { } } }); + state.fields + .querySelectorAll(".row-edit-binary-control") + .forEach(function (binaryControl) { + revokeRowEditBinaryPreview(binaryControl); + }); state.fields.innerHTML = ""; } @@ -4241,34 +5410,50 @@ function createRowEditField(column, value, isPk, columnType, index, options) { controlWrap.className = "row-edit-control-wrap"; var context = columnFormControlContext(column, isPk, columnType, options); - var pluginControl = makeColumnField(options.manager, context); + var isBinaryField = shouldUseBinaryControl(value, options); + var pluginControl = isBinaryField + ? null + : makeColumnField(options.manager, context); var useTextarea = - (pluginControl && pluginControl.useTextarea === true) || - shouldUseTextarea(value, columnType); + !isBinaryField && + ((pluginControl && pluginControl.useTextarea === true) || + shouldUseTextarea(value, columnType)); var control = useTextarea ? document.createElement("textarea") : document.createElement("input"); + var initialValue = isBinaryField + ? binaryEncodedValue(value) + : valueToEditText(value); + var initialValueKind = options.valueKind || rowEditValueKind(value); + if (isBinaryField) { + initialValueKind = isBase64JsonValue(value) ? "binary" : "null"; + } control.className = "row-edit-input"; control.id = fieldId; control.name = column; - control.value = valueToEditText(value); + control.value = initialValue; control.setAttribute("aria-describedby", metaId); - control.dataset.initialValue = valueToEditText(value); - control.dataset.initialValueKind = - options.valueKind || rowEditValueKind(value); + control.dataset.initialValue = initialValue; + control.dataset.initialValueKind = initialValueKind; control.dataset.primaryKey = isPk ? "1" : "0"; control.dataset.currentValueKind = control.dataset.initialValueKind; + if (isBinaryField) { + control.dataset.binaryField = "1"; + control.dataset.notNull = options.notnull ? "1" : "0"; + } if (hasDefaultExpression) { control.dataset.useSqliteDefault = useSqliteDefault ? "1" : "0"; } if (useSqliteDefault) { control.disabled = true; } - if (options.omitIfBlank) { + if (options.omitIfBlank || (isBinaryField && options.mode === "insert")) { control.dataset.omitIfBlank = "1"; } - if (control.nodeName === "TEXTAREA") { + if (isBinaryField) { + control.type = "hidden"; + } else if (control.nodeName === "TEXTAREA") { control.rows = Math.min(8, Math.max(3, control.value.split("\n").length)); } else { control.type = "text"; @@ -4342,9 +5527,11 @@ function createRowEditField(column, value, isPk, columnType, index, options) { field._datasetteColumnFormField = fieldApi; var pluginControlElement = renderColumnField(pluginControl, fieldApi); var controlElement = + (isBinaryField && + createRowEditBinaryControlElement(control, value, options, labelId)) || pluginControlElement || rowEditControlElement(control, options.autocompleteUrl); - if (options.autocompleteUrl && !pluginControlElement) { + if (options.autocompleteUrl && !pluginControlElement && !isBinaryField) { control.addEventListener("input", function () { setForeignKeyMetaLink(meta, options.autocompleteUrl, null); }); @@ -4435,18 +5622,64 @@ function clearRowEditDialogError(state) { state.error.textContent = ""; } -function showRowEditDialogError(state, message) { +function showRowEditDialogError(state, message, options) { state.error.hidden = false; state.error.textContent = message; - state.error.focus(); + if (!options || options.focus !== false) { + state.error.focus(); + } +} + +function rowEditIsMultipleInsert(state) { + return state.mode === "insert" && state.insertMode === "multiple"; +} + +function syncRowEditInsertModeUi(state) { + var isInsert = state.mode === "insert"; + var isMultiple = rowEditIsMultipleInsert(state); + state.fields.hidden = isMultiple; + state.bulkInsertPanel.hidden = !isMultiple; + state.bulkInsertEditor.hidden = !isMultiple || state.bulkInsertPreviewReady; + state.bulkInsertPreview.hidden = !isMultiple || !state.bulkInsertPreviewReady; + state.bulkInsertLink.hidden = !isInsert || isMultiple; + state.singleInsertLink.hidden = !isInsert || !isMultiple; } function updateRowEditDialogButtons(state) { state.saveButton.disabled = - state.isLoading || state.isSaving || !state.hasLoaded; + state.isLoading || + state.isSaving || + !state.hasLoaded || + state.bulkInsertInserted || + (rowEditIsMultipleInsert(state) && + !state.bulkInsertPreviewReady && + !!state.bulkInsertLiveValidationError); state.cancelButton.disabled = state.isSaving; - var saveLabel = state.mode === "insert" ? "Insert row" : "Save"; - state.saveButton.textContent = state.isSaving ? "Saving..." : saveLabel; + syncRowEditInsertModeUi(state); + state.cancelButton.textContent = + rowEditIsMultipleInsert(state) && + state.bulkInsertPreviewReady && + !state.bulkInsertInserted + ? "Back" + : state.bulkInsertInserted + ? "Close and view table" + : "Cancel"; + var saveLabel = rowEditIsMultipleInsert(state) + ? state.bulkInsertInserted + ? "Inserted" + : state.bulkInsertPreviewReady + ? bulkInsertSaveLabel(state) + : "Preview rows" + : state.mode === "insert" + ? "Insert row" + : "Save"; + state.saveButton.textContent = state.isSaving + ? rowEditIsMultipleInsert(state) + ? bulkInsertConflictMode(state) === "upsert" + ? "Updating..." + : "Inserting..." + : "Saving..." + : saveLabel; state.form.setAttribute( "aria-busy", state.isLoading || state.isSaving ? "true" : "false", @@ -4466,6 +5699,9 @@ function setRowEditDialogSaving(state, isSaving) { function valueFromRowEditControl(control) { var value = control.value; + if (rowEditControlValueKind(control) === "binary") { + return rowEditBinaryValue(value); + } return valueFromRowEditText( control.name, value, @@ -4476,6 +5712,9 @@ function valueFromRowEditControl(control) { function valueFromRowEditText(name, value, initialValueKind) { var trimmed = value.trim(); + if (initialValueKind === "binary") { + return rowEditBinaryValue(value); + } if (initialValueKind === "null" && value === "") { return null; } @@ -4603,7 +5842,11 @@ function collectRowFormValues(state) { if (control.dataset.useSqliteDefault === "1") { return; } - if (control.dataset.omitIfBlank === "1" && control.value === "") { + if ( + control.dataset.omitIfBlank === "1" && + control.value === "" && + rowEditControlValueKind(control) !== "binary" + ) { return; } if ( @@ -4637,6 +5880,16 @@ function rowEditDialogHasChanges(state) { if (!state || !state.hasLoaded || state.isLoading) { return false; } + if (state.bulkInsertInserted) { + return false; + } + if ( + state.mode === "insert" && + state.bulkInsertTextarea && + state.bulkInsertTextarea.value.trim() + ) { + return true; + } var fields = state.fields.querySelectorAll(".row-edit-field"); for (var i = 0; i < fields.length; i += 1) { var fieldApi = fields[i]._datasetteColumnFormField; @@ -4670,6 +5923,831 @@ function closeRowEditDialogIfConfirmed(state) { return true; } +function setRowInsertDialogTitle(state) { + var insertData = tableInsertData() || {}; + var title = rowEditIsMultipleInsert(state) + ? "Insert multiple rows" + : "Insert row"; + setRowDialogTitle( + state.title, + insertData.tableName ? title + " into " + insertData.tableName : title, + ); +} + +function showMultipleRowInsert(state) { + if (!state || state.mode !== "insert" || state.isSaving) { + return; + } + state.insertMode = "multiple"; + if (state.bulkInsertPreviewReady || state.bulkInsertInserted) { + resetBulkInsertPreview(state); + } + clearRowEditDialogError(state); + setRowInsertDialogTitle(state); + syncBulkInsertConflictUi(state); + syncBulkInsertTextareaValidation(state); + updateRowEditDialogButtons(state); + state.bulkInsertTextarea.focus(); +} + +function showSingleRowInsert(state) { + if (!state || state.mode !== "insert" || state.isSaving) { + return; + } + state.insertMode = "single"; + clearRowEditDialogError(state); + setRowInsertDialogTitle(state); + updateRowEditDialogButtons(state); + if (!focusFirstRowEditControl(state, { skipReadonly: true })) { + state.saveButton.focus(); + } +} + +function bulkInsertConflictMode(state) { + if (!state || !state.bulkInsertHasPrimaryKeyColumns) { + return "insert"; + } + return state.bulkInsertConflictMode || "ignore"; +} + +function syncBulkInsertConflictUi(state) { + if (!state || !state.bulkInsertConflictField) { + return; + } + var insertData = tableInsertData() || {}; + var primaryKeys = insertData.primaryKeys || []; + var hasPrimaryKeys = primaryKeys.length > 0; + var hasPrimaryKeyColumns = + hasPrimaryKeys && + bulkInsertTextIncludesPrimaryKeyColumns( + state.bulkInsertTextarea.value, + state.bulkInsertColumnDetails, + primaryKeys, + ); + state.bulkInsertHasPrimaryKeyColumns = hasPrimaryKeyColumns; + var canUpsert = hasPrimaryKeyColumns && !!state.currentUpsertUrl; + var upsertOption = state.bulkInsertConflictSelect.querySelector( + 'option[value="upsert"]', + ); + if (upsertOption) { + upsertOption.disabled = !canUpsert; + upsertOption.hidden = !canUpsert; + } + if ( + !hasPrimaryKeyColumns || + (!canUpsert && bulkInsertConflictMode(state) === "upsert") + ) { + state.bulkInsertConflictMode = hasPrimaryKeys ? "ignore" : "insert"; + state.bulkInsertConflictSelect.value = state.bulkInsertConflictMode; + } + state.bulkInsertConflictField.hidden = !hasPrimaryKeyColumns; + state.bulkInsertConflictSelect.value = bulkInsertConflictMode(state); + var helpText = ""; + if (bulkInsertConflictMode(state) === "upsert") { + helpText = + "Rows with existing primary keys will be updated; new primary keys will be inserted."; + } else if (bulkInsertConflictMode(state) === "ignore") { + helpText = "Rows with existing primary keys will be skipped."; + } else { + helpText = "Rows with existing primary keys will stop the import."; + } + state.bulkInsertConflictHelp.textContent = helpText; +} + +function bulkInsertSaveLabel(state) { + if (!state.bulkInsertPreviewReady) { + return "Preview rows"; + } + if (bulkInsertConflictMode(state) === "upsert") { + return "Update or insert rows"; + } + return "Insert these rows"; +} + +function readTextFile(file) { + if (file.text) { + return file.text(); + } + return new Promise(function (resolve, reject) { + var reader = new FileReader(); + reader.onload = function () { + resolve(reader.result || ""); + }; + reader.onerror = function () { + reject(reader.error); + }; + reader.readAsText(file); + }); +} + +async function loadBulkInsertTextFile(state, file) { + if (!file) { + return; + } + try { + state.bulkInsertTextarea.value = await readTextFile(file); + state.bulkInsertTextarea.dispatchEvent( + new Event("input", { bubbles: true }), + ); + state.bulkInsertTextarea.focus(); + } catch (_error) { + showRowEditDialogError(state, "Could not read that text file."); + } +} + +function bulkInsertTemplateText(state) { + return (state.bulkInsertTemplateColumns || []).join("\t"); +} + +async function copyTextToClipboard(text) { + if (navigator.clipboard && navigator.clipboard.writeText) { + await navigator.clipboard.writeText(text); + return; + } + var textarea = document.createElement("textarea"); + textarea.value = text; + textarea.setAttribute("readonly", ""); + textarea.style.position = "fixed"; + textarea.style.top = "-1000px"; + textarea.style.left = "-1000px"; + document.body.appendChild(textarea); + textarea.select(); + var copied = document.execCommand("copy"); + textarea.remove(); + if (!copied) { + throw new Error("copy failed"); + } +} + +function setBulkInsertCopyButtonReady(state) { + state.copyTemplateButton.textContent = ""; + var wideLabel = document.createElement("span"); + wideLabel.className = "row-edit-copy-template-label-wide"; + wideLabel.textContent = "Copy spreadsheet template"; + state.copyTemplateButton.appendChild(wideLabel); + var narrowLabel = document.createElement("span"); + narrowLabel.className = "row-edit-copy-template-label-narrow"; + narrowLabel.textContent = "Copy template"; + state.copyTemplateButton.appendChild(narrowLabel); +} + +function setBulkInsertCopyButtonCopied(state) { + state.copyTemplateButton.textContent = "Copied"; + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = setTimeout(function () { + setBulkInsertCopyButtonReady(state); + }, 1500); +} + +function resetBulkInsertPreview(state) { + state.bulkInsertPreviewRows = null; + state.bulkInsertPreviewReady = false; + state.bulkInsertInserted = false; + state.bulkInsertInsertedCount = 0; + state.bulkInsertPreview.hidden = true; + state.bulkInsertPreview.textContent = ""; + state.bulkInsertProgress.hidden = true; + state.bulkInsertProgressBar.value = 0; + state.bulkInsertProgressBar.max = 1; + state.bulkInsertProgressStatus.textContent = ""; + syncBulkInsertConflictUi(state); + syncRowEditInsertModeUi(state); +} + +function normalizeBulkInsertCell(column, value) { + if (typeof value === "undefined") { + return column.notnull ? "" : null; + } + if (value === null) { + return column.notnull ? "" : null; + } + if (value === "" && column.notnull) { + return ""; + } + if (column.value_kind === "number" && typeof value === "string") { + return valueFromRowEditText(column.name, value, "number"); + } + return value; +} + +function rowObjectForBulkInsert(valuesByColumn, columns) { + var row = {}; + columns.forEach(function (column) { + var hasValue = Object.prototype.hasOwnProperty.call( + valuesByColumn, + column.name, + ); + if (!hasValue) { + return; + } + row[column.name] = normalizeBulkInsertCell( + column, + valuesByColumn[column.name], + ); + }); + return row; +} + +function splitDelimitedRows(text, delimiter) { + var rows = []; + var row = []; + var cell = ""; + var inQuotes = false; + + for (var i = 0; i < text.length; i += 1) { + var character = text[i]; + if (inQuotes) { + if (character === '"') { + if (text[i + 1] === '"') { + cell += '"'; + i += 1; + } else { + inQuotes = false; + } + } else { + cell += character; + } + continue; + } + + if (character === '"') { + inQuotes = true; + } else if (character === delimiter) { + row.push(cell); + cell = ""; + } else if (character === "\n" || character === "\r") { + row.push(cell); + rows.push(row); + row = []; + cell = ""; + if (character === "\r" && text[i + 1] === "\n") { + i += 1; + } + } else { + cell += character; + } + } + + if (inQuotes) { + throw new Error("Unclosed quoted value."); + } + row.push(cell); + rows.push(row); + + while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { + rows.pop(); + } + return rows; +} + +function bulkInsertDelimitedRowIsBlank(row) { + return row.every(function (value) { + return value.trim() === ""; + }); +} + +function delimiterPreviewRows(text, delimiter) { + try { + return splitDelimitedRows(text, delimiter); + } catch (_error) { + return []; + } +} + +function splitSingleColumnRows(text) { + var rows = text.split(/\r\n|\n|\r/).map(function (line) { + return [line]; + }); + while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { + rows.pop(); + } + return rows; +} + +function detectBulkInsertDelimiter(text, columns) { + var firstLine = + text.split(/\r\n|\n|\r/).find(function (line) { + return line.trim() !== ""; + }) || ""; + var csvRows = delimiterPreviewRows(firstLine, ","); + var tsvRows = delimiterPreviewRows(firstLine, "\t"); + var csvColumns = csvRows.length ? csvRows[0].length : 0; + var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; + + if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { + return "\t"; + } + if (tsvColumns > csvColumns) { + return "\t"; + } + if (csvColumns > 1) { + return ","; + } + if (tsvColumns > 1) { + return "\t"; + } + if (columns.length === 1 || bulkInsertColumnMap(columns)[firstLine.trim()]) { + return null; + } + throw new Error("Could not detect CSV or TSV columns."); +} + +function bulkInsertColumnMap(columns) { + var map = {}; + columns.forEach(function (column) { + map[column.name] = column; + }); + return map; +} + +function bulkInsertTextIncludesPrimaryKeyColumns(text, columns, primaryKeys) { + if (!primaryKeys.length || !text.trim()) { + return false; + } + var trimmed = text.trim(); + try { + if (trimmed[0] === "[" || trimmed[0] === "{") { + return jsonBulkInsertTextIncludesPrimaryKeyColumns(trimmed, primaryKeys); + } + return delimitedBulkInsertTextIncludesPrimaryKeyColumns( + trimmed, + columns, + primaryKeys, + ); + } catch (_error) { + return false; + } +} + +function jsonBulkInsertTextIncludesPrimaryKeyColumns(text, primaryKeys) { + var rows = parseJsonObjectRows(text); + var seenKeys = {}; + rows.forEach(function (row) { + Object.keys(row).forEach(function (key) { + seenKeys[key] = true; + }); + }); + return primaryKeys.every(function (key) { + return !!seenKeys[key]; + }); +} + +function delimitedBulkInsertTextIncludesPrimaryKeyColumns( + text, + columns, + primaryKeys, +) { + var delimiter = detectBulkInsertDelimiter(text, columns); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + return false; + } + + var columnMap = bulkInsertColumnMap(columns); + var header = rows[0].map(function (value) { + return value.trim(); + }); + var headerMatches = header.filter(function (name) { + return !!columnMap[name]; + }).length; + if (headerMatches > 0) { + return primaryKeys.every(function (key) { + return header.indexOf(key) !== -1; + }); + } + + var headers = columns.map(function (column) { + return column.name; + }); + var suppliedColumnCount = rows.reduce(function (count, row) { + return Math.max(count, row.length); + }, 0); + return primaryKeys.every(function (key) { + var index = headers.indexOf(key); + return index !== -1 && index < suppliedColumnCount; + }); +} + +function bulkInsertLiveValidationShouldWait(message) { + return ( + message === "Paste rows before previewing." || + message === "No data rows found to preview." || + message.indexOf("Invalid JSON:") === 0 + ); +} + +function bulkInsertLiveValidationError(state) { + var text = state.bulkInsertTextarea.value; + if (!text.trim()) { + return null; + } + try { + parseBulkInsertRows(text, state.bulkInsertColumnDetails); + } catch (error) { + var message = error.message || "Could not preview rows."; + return bulkInsertLiveValidationShouldWait(message) ? null : message; + } + return null; +} + +function syncBulkInsertTextareaValidation(state) { + if (!rowEditIsMultipleInsert(state) || state.bulkInsertPreviewReady) { + state.bulkInsertLiveValidationError = null; + return; + } + state.bulkInsertLiveValidationError = bulkInsertLiveValidationError(state); + if (state.bulkInsertLiveValidationError) { + showRowEditDialogError(state, state.bulkInsertLiveValidationError, { + focus: false, + }); + } else { + clearRowEditDialogError(state); + } +} + +function parseJsonBulkInsertRows(text, columns) { + var parsed = parseJsonObjectRows(text); + + var columnMap = bulkInsertColumnMap(columns); + return parsed.map(function (item, index) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("JSON row " + (index + 1) + " must be an object."); + } + Object.keys(item).forEach(function (key) { + if (!columnMap[key]) { + throw new Error( + "JSON row " + (index + 1) + " has unknown column " + key + ".", + ); + } + }); + return rowObjectForBulkInsert(item, columns); + }); +} + +function parseDelimitedBulkInsertRows(text, columns) { + var delimiter = detectBulkInsertDelimiter(text, columns); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + + var columnMap = bulkInsertColumnMap(columns); + var header = rows[0].map(function (value) { + return value.trim(); + }); + var headerMatches = header.filter(function (name) { + return !!columnMap[name]; + }).length; + var hasHeader = headerMatches > 0; + var dataRows = hasHeader ? rows.slice(1) : rows; + var headers = hasHeader + ? header + : columns.map(function (column) { + return column.name; + }); + var seenHeaders = {}; + + if (hasHeader) { + headers.forEach(function (name) { + if (!name) { + return; + } + if (!columnMap[name]) { + throw new Error("Unknown column " + name + " in header row."); + } + if (seenHeaders[name]) { + throw new Error("Duplicate column " + name + " in header row."); + } + seenHeaders[name] = true; + }); + } + + if (!dataRows.length) { + throw new Error("No data rows found to preview."); + } + + return dataRows.map(function (row, rowIndex) { + if (row.length > headers.length) { + throw new Error( + "Row " + + (rowIndex + 1) + + " has " + + row.length + + " values, but only " + + headers.length + + " columns were provided.", + ); + } + var valuesByColumn = {}; + row.forEach(function (value, index) { + var columnName = headers[index]; + if (columnMap[columnName]) { + valuesByColumn[columnName] = value; + } + }); + return rowObjectForBulkInsert(valuesByColumn, columns); + }); +} + +function parseBulkInsertRows(text, columns) { + var trimmed = text.trim(); + if (!trimmed) { + throw new Error("Paste rows before previewing."); + } + if (trimmed[0] === "[" || trimmed[0] === "{") { + return parseJsonBulkInsertRows(trimmed, columns); + } + return parseDelimitedBulkInsertRows(trimmed, columns); +} + +function bulkInsertPreviewValue(value) { + if (value === null) { + return "null"; + } + if (typeof value === "object") { + return JSON.stringify(value); + } + return String(value); +} + +function bulkInsertPreviewCell(column, hasValue, value) { + if (!hasValue && column.is_auto_pk) { + return { + text: "auto", + className: "row-edit-bulk-preview-auto", + }; + } + if (value === null) { + return { + text: bulkInsertPreviewValue(value), + className: "row-edit-bulk-preview-null", + }; + } + return { + text: hasValue ? bulkInsertPreviewValue(value) : "", + className: "", + }; +} + +function renderBulkInsertPreview(state, rows) { + state.bulkInsertPreview.textContent = ""; + var summary = document.createElement("p"); + summary.className = "row-edit-bulk-preview-summary"; + summary.textContent = + "Previewing " + rows.length + " row" + (rows.length === 1 ? "." : "s."); + state.bulkInsertPreview.appendChild(summary); + + var tableWrap = document.createElement("div"); + tableWrap.className = "row-edit-bulk-preview-table-wrap"; + var table = document.createElement("table"); + table.className = "row-edit-bulk-preview-table"; + var thead = document.createElement("thead"); + var headerRow = document.createElement("tr"); + state.bulkInsertColumnDetails.forEach(function (column) { + var th = document.createElement("th"); + th.scope = "col"; + th.textContent = column.name; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + var tbody = document.createElement("tbody"); + rows.forEach(function (row) { + var tr = document.createElement("tr"); + state.bulkInsertColumnDetails.forEach(function (column) { + var td = document.createElement("td"); + var hasValue = Object.prototype.hasOwnProperty.call(row, column.name); + var value = hasValue ? row[column.name] : ""; + var cell = bulkInsertPreviewCell(column, hasValue, value); + td.textContent = cell.text; + if (cell.className) { + td.className = cell.className; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + tableWrap.appendChild(table); + state.bulkInsertPreview.appendChild(tableWrap); + state.bulkInsertPreview.hidden = false; +} + +function previewBulkInsertRows(state) { + clearRowEditDialogError(state); + resetBulkInsertPreview(state); + syncBulkInsertConflictUi(state); + try { + var rows = parseBulkInsertRows( + state.bulkInsertTextarea.value, + state.bulkInsertColumnDetails, + ); + state.bulkInsertPreviewRows = rows; + state.bulkInsertPreviewReady = true; + renderBulkInsertPreview(state, rows); + updateRowEditDialogButtons(state); + } catch (error) { + showRowEditDialogError(state, error.message || "Could not preview rows."); + updateRowEditDialogButtons(state); + } +} + +function updateBulkInsertProgress(state, inserted, total) { + var words = bulkInsertProgressWords(state); + state.bulkInsertProgress.hidden = false; + state.bulkInsertProgressBar.max = total || 1; + state.bulkInsertProgressBar.value = inserted; + state.bulkInsertProgressStatus.textContent = + inserted >= total + ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." + : words.active + " " + inserted + " of " + total + " rows..."; +} + +function bulkInsertBatches(rows, batchSize) { + var batches = []; + var size = Math.max(1, batchSize || 1); + for (var index = 0; index < rows.length; index += size) { + batches.push(rows.slice(index, index + size)); + } + return batches; +} + +function animateBulkInsertProgress(state, from, to, total, duration) { + state.bulkInsertProgress.hidden = false; + state.bulkInsertProgressBar.max = total || 1; + if (duration <= 0 || !window.requestAnimationFrame) { + updateBulkInsertProgress(state, to, total); + return Promise.resolve(); + } + + return new Promise(function (resolve) { + var startTime = null; + var step = function (timestamp) { + if (startTime === null) { + startTime = timestamp; + } + var progress = Math.min((timestamp - startTime) / duration, 1); + var easedProgress = 1 - Math.pow(1 - progress, 3); + var value = from + (to - from) * easedProgress; + var displayValue = progress === 1 ? to : Math.floor(value); + var words = bulkInsertProgressWords(state); + state.bulkInsertProgressBar.value = value; + state.bulkInsertProgressStatus.textContent = + displayValue >= total + ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." + : words.active + " " + displayValue + " of " + total + " rows..."; + if (progress < 1) { + window.requestAnimationFrame(step); + } else { + updateBulkInsertProgress(state, to, total); + resolve(); + } + }; + window.requestAnimationFrame(step); + }); +} + +function bulkInsertProgressWords(state) { + var mode = bulkInsertConflictMode(state); + if (mode === "upsert") { + return { + active: "Upserting", + complete: "upserted", + }; + } + if (mode === "ignore") { + return { + active: "Processing", + complete: "processed", + }; + } + return { + active: "Inserting", + complete: "inserted", + }; +} + +function validateBulkInsertConflictRows(state, rows) { + if (bulkInsertConflictMode(state) !== "upsert") { + return null; + } + var insertData = tableInsertData() || {}; + var primaryKeys = insertData.primaryKeys || []; + for (var index = 0; index < rows.length; index += 1) { + var row = rows[index]; + var missing = primaryKeys.filter(function (key) { + return ( + !Object.prototype.hasOwnProperty.call(row, key) || + row[key] === null || + typeof row[key] === "undefined" + ); + }); + if (missing.length) { + return ( + "Row " + + (index + 1) + + " is missing primary key " + + missing.join(", ") + + ". Upsert requires primary key values for every row." + ); + } + } + return null; +} + +async function insertBulkPreviewRows(state) { + if (!state.bulkInsertPreviewRows || state.bulkInsertInserted) { + return; + } + var conflictMode = bulkInsertConflictMode(state); + var url = + conflictMode === "upsert" ? state.currentUpsertUrl : state.currentInsertUrl; + if (!url) { + showRowEditDialogError( + state, + conflictMode === "upsert" + ? "Could not find the row upsert URL." + : "Could not find the row insert URL.", + ); + return; + } + + var rows = state.bulkInsertPreviewRows; + var validationError = validateBulkInsertConflictRows(state, rows); + if (validationError) { + showRowEditDialogError(state, validationError); + return; + } + var total = rows.length; + var inserted = state.bulkInsertInsertedCount || 0; + var batches = bulkInsertBatches( + rows.slice(inserted), + state.bulkInsertMaxRows, + ); + var progressAnimationDuration = 500 / Math.max(batches.length, 1); + + clearRowEditDialogError(state); + updateBulkInsertProgress(state, inserted, total); + setRowEditDialogSaving(state, true); + try { + for (var batchIndex = 0; batchIndex < batches.length; batchIndex += 1) { + var batch = batches[batchIndex]; + var payload = { rows: batch }; + if (conflictMode === "ignore") { + payload.ignore = true; + } + var response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = null; + try { + data = await response.json(); + } catch (_error) { + data = null; + } + if (!response.ok || (data && data.ok === false)) { + throw rowMutationRequestError(response, data); + } + var previousInserted = inserted; + inserted += batch.length; + state.bulkInsertInsertedCount = inserted; + await animateBulkInsertProgress( + state, + previousInserted, + inserted, + total, + progressAnimationDuration, + ); + } + state.bulkInsertInserted = true; + state.shouldReloadOnClose = true; + state.redirectOnCloseUrl = tableBaseUrl().toString(); + updateBulkInsertProgress(state, inserted, total); + } catch (error) { + showRowEditDialogError(state, error.message || "Could not insert rows."); + } finally { + setRowEditDialogSaving(state, false); + } +} + function scheduleCloseRowEditDialogIfConfirmed(state) { // Fix for an issue in Safari where hitting Esc would show // the confirm() prompt asking if state should be discarded @@ -4768,6 +6846,14 @@ async function saveRowEditDialog(state) { if (state.isLoading || state.isSaving || !state.hasLoaded) { return; } + if (rowEditIsMultipleInsert(state)) { + if (!state.bulkInsertPreviewReady) { + previewBulkInsertRows(state); + } else if (!state.bulkInsertInserted) { + await insertBulkPreviewRows(state); + } + return; + } clearRowEditDialogError(state); setRowEditDialogSaving(state, true); @@ -4925,9 +7011,12 @@ function renderRowEditFields(state, data) { var columns = data.columns || (row ? Object.keys(row) : []); var primaryKeys = data.primary_keys || []; var columnTypes = data.column_types || {}; + var columnDetails = data.column_details || {}; + state.insertMode = "single"; destroyRowEditFields(state); columns.forEach(function (column, index) { + var columnDetail = columnDetails[column] || {}; state.fields.appendChild( createRowEditField( column, @@ -4941,7 +7030,9 @@ function renderRowEditFields(state, data) { form: state.form, manager: state.manager, mode: state.mode, + notnull: columnDetail.notnull, primaryKeyReadonly: true, + sqliteType: columnDetail.sqlite_type, }, ), ); @@ -4956,7 +7047,23 @@ function renderRowEditFields(state, data) { function renderRowInsertFields(state, data) { var columns = data.columns || []; + var bulkColumns = data.bulkColumns || columns; + state.insertMode = "single"; + state.bulkInsertColumnDetails = bulkColumns.slice(); + state.bulkInsertMaxRows = data.maxInsertRows || 100; + state.bulkInsertColumns = bulkColumns.map(function (column) { + return column.name; + }); + state.bulkInsertTemplateColumns = columns.map(function (column) { + return column.name; + }); + state.copyTemplateButton.disabled = !state.bulkInsertTemplateColumns.length; + setBulkInsertCopyButtonReady(state); + syncBulkInsertConflictUi(state); + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = null; + resetBulkInsertPreview(state); destroyRowEditFields(state); columns.forEach(function (column, index) { state.fields.appendChild( @@ -5046,7 +7153,36 @@ function ensureRowEditDialog(manager) {

Loading row...

+
+ +
@@ -5062,6 +7198,27 @@ function ensureRowEditDialog(manager) { loading: dialog.querySelector(".row-edit-loading"), error: dialog.querySelector(".row-edit-error"), fields: dialog.querySelector(".row-edit-fields"), + bulkInsertPanel: dialog.querySelector(".row-edit-bulk"), + bulkInsertEditor: dialog.querySelector(".row-edit-bulk-editor"), + bulkInsertTextarea: dialog.querySelector(".row-edit-bulk-textarea"), + bulkInsertPreview: dialog.querySelector(".row-edit-bulk-preview"), + bulkInsertProgress: dialog.querySelector(".row-edit-bulk-progress"), + bulkInsertProgressBar: dialog.querySelector(".row-edit-bulk-progress-bar"), + bulkInsertProgressStatus: dialog.querySelector( + ".row-edit-bulk-progress-status", + ), + bulkInsertConflictField: dialog.querySelector(".row-edit-bulk-conflict"), + bulkInsertConflictSelect: dialog.querySelector( + ".row-edit-bulk-conflict-mode", + ), + bulkInsertConflictHelp: dialog.querySelector( + ".row-edit-bulk-conflict-help", + ), + copyTemplateButton: dialog.querySelector(".row-edit-copy-template"), + bulkInsertOpenFileButton: dialog.querySelector(".row-edit-bulk-open-file"), + bulkInsertFileInput: dialog.querySelector(".row-edit-bulk-file-input"), + bulkInsertLink: dialog.querySelector(".row-edit-bulk-insert"), + singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), currentButton: null, @@ -5069,9 +7226,25 @@ function ensureRowEditDialog(manager) { currentRowId: null, currentPkPath: null, currentInsertUrl: null, + currentUpsertUrl: null, currentUpdateUrl: null, currentFragmentUrl: null, mode: "edit", + insertMode: "single", + bulkInsertConflictMode: "ignore", + bulkInsertHasPrimaryKeyColumns: false, + bulkInsertLiveValidationError: null, + bulkInsertColumns: [], + bulkInsertTemplateColumns: [], + bulkInsertColumnDetails: [], + bulkInsertPreviewRows: null, + bulkInsertPreviewReady: false, + bulkInsertInserted: false, + bulkInsertInsertedCount: 0, + bulkInsertMaxRows: 100, + shouldReloadOnClose: false, + redirectOnCloseUrl: null, + copyTemplateResetTimer: null, loadId: 0, manager: manager, isLoading: false, @@ -5087,12 +7260,139 @@ function ensureRowEditDialog(manager) { }); rowEditDialogState.cancelButton.addEventListener("click", function () { + if ( + rowEditIsMultipleInsert(rowEditDialogState) && + rowEditDialogState.bulkInsertPreviewReady && + !rowEditDialogState.bulkInsertInserted && + !rowEditDialogState.isSaving + ) { + resetBulkInsertPreview(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + rowEditDialogState.bulkInsertTextarea.focus(); + return; + } if (!rowEditDialogState.isSaving) { rowEditDialogState.shouldRestoreFocus = true; dialog.close(); } }); + rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { + ev.preventDefault(); + showMultipleRowInsert(rowEditDialogState); + }); + + rowEditDialogState.singleInsertLink.addEventListener("click", function (ev) { + ev.preventDefault(); + showSingleRowInsert(rowEditDialogState); + }); + + rowEditDialogState.copyTemplateButton.addEventListener( + "click", + async function () { + try { + await copyTextToClipboard(bulkInsertTemplateText(rowEditDialogState)); + clearRowEditDialogError(rowEditDialogState); + setBulkInsertCopyButtonCopied(rowEditDialogState); + } catch (_error) { + showRowEditDialogError( + rowEditDialogState, + "Could not copy the spreadsheet template.", + ); + } + }, + ); + + rowEditDialogState.bulkInsertOpenFileButton.addEventListener( + "click", + function () { + rowEditDialogState.bulkInsertFileInput.click(); + }, + ); + + rowEditDialogState.bulkInsertFileInput.addEventListener( + "change", + async function (ev) { + var files = ev.target.files; + await loadBulkInsertTextFile( + rowEditDialogState, + files && files.length ? files[0] : null, + ); + ev.target.value = ""; + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragenter", + function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.add( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragover", + function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.add( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragleave", + function () { + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "drop", + async function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + var files = ev.dataTransfer && ev.dataTransfer.files; + if (!files || !files.length) { + return; + } + await loadBulkInsertTextFile(rowEditDialogState, files[0]); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragend", + function () { + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener("input", function () { + resetBulkInsertPreview(rowEditDialogState); + syncBulkInsertTextareaValidation(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + }); + + rowEditDialogState.bulkInsertConflictSelect.addEventListener( + "change", + function () { + rowEditDialogState.bulkInsertConflictMode = + rowEditDialogState.bulkInsertConflictSelect.value; + syncBulkInsertConflictUi(rowEditDialogState); + resetBulkInsertPreview(rowEditDialogState); + syncBulkInsertTextareaValidation(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + }, + ); + dialog.addEventListener("click", function (ev) { if (ev.target === dialog) { closeRowEditDialogIfConfirmed(rowEditDialogState); @@ -5114,8 +7414,17 @@ function ensureRowEditDialog(manager) { dialog.addEventListener("close", function () { var state = rowEditDialogState; + var shouldReloadOnClose = state.shouldReloadOnClose; + var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; state.isClosePending = false; + state.bulkInsertLiveValidationError = null; + state.shouldReloadOnClose = false; + state.redirectOnCloseUrl = null; + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = null; + setBulkInsertCopyButtonReady(state); + resetBulkInsertPreview(state); clearRowEditDialogError(state); state.hasLoaded = false; destroyRowEditFields(state); @@ -5128,6 +7437,13 @@ function ensureRowEditDialog(manager) { ) { state.currentButton.focus(); } + if (shouldReloadOnClose) { + if (redirectOnCloseUrl) { + location.href = redirectOnCloseUrl; + } else { + location.reload(); + } + } }); return rowEditDialogState; @@ -5150,8 +7466,10 @@ async function openRowEditDialog(button, manager) { state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); state.currentInsertUrl = null; + state.currentUpsertUrl = null; state.currentUpdateUrl = rowUpdateUrl(row); state.currentFragmentUrl = rowFragmentUrl(row); + state.insertMode = "single"; if (state.currentUpdateUrl) { state.form.action = new URL( state.currentUpdateUrl, @@ -5177,6 +7495,7 @@ async function openRowEditDialog(button, manager) { ); state.summary.hidden = true; state.summary.textContent = ""; + syncRowEditInsertModeUi(state); if (!state.dialog.open) { state.dialog.showModal(); @@ -5225,8 +7544,16 @@ function openRowInsertDialog(button, manager) { state.currentRowId = null; state.currentPkPath = null; state.currentInsertUrl = tableInsertUrl(); + state.currentUpsertUrl = tableUpsertUrl(); state.currentUpdateUrl = null; state.currentFragmentUrl = null; + state.insertMode = "single"; + state.bulkInsertConflictMode = "ignore"; + state.bulkInsertLiveValidationError = null; + state.bulkInsertTextarea.value = ""; + state.shouldReloadOnClose = false; + state.redirectOnCloseUrl = null; + resetBulkInsertPreview(state); state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; @@ -5244,14 +7571,10 @@ function openRowInsertDialog(button, manager) { setRowEditDialogLoading(state, false); destroyRowEditFields(state); state.dialog.removeAttribute("aria-describedby"); - setRowDialogTitle( - state.title, - insertData.tableName - ? "Insert row into " + insertData.tableName - : "Insert row", - ); + setRowInsertDialogTitle(state); state.summary.hidden = true; state.summary.textContent = ""; + syncRowEditInsertModeUi(state); if (!state.dialog.open) { state.dialog.showModal(); @@ -5287,333 +7610,6 @@ function initRowInsertActions(manager) { }); } -function clearTableLabelColumnsDialogError(state) { - state.error.hidden = true; - state.error.textContent = ""; - state.dialog.removeAttribute("aria-describedby"); -} - -function showTableLabelColumnsDialogError(state, message) { - state.error.hidden = false; - state.error.textContent = message; - state.dialog.setAttribute("aria-describedby", "table-label-columns-error"); - state.error.focus(); -} - -function setTableLabelColumnsDialogBusy(state, isBusy) { - state.isBusy = isBusy; - state.cancelButton.disabled = isBusy; - state.saveButton.disabled = isBusy; - state.resetButton.disabled = isBusy || !state.isOverridden; - state.saveButton.textContent = isBusy ? "Saving..." : "Save"; - state.list - .querySelectorAll("input, button") - .forEach(function (control) { - control.disabled = isBusy; - }); - if (!isBusy) { - updateTableLabelColumnsMoveButtons(state); - state.resetButton.disabled = !state.isOverridden; - } -} - -function updateTableLabelColumnsMoveButtons(state) { - var rows = Array.prototype.slice.call( - state.list.querySelectorAll(".table-label-columns-row"), - ); - rows.forEach(function (row, index) { - row.querySelector(".table-label-columns-move-up").disabled = - state.isBusy || index === 0; - row.querySelector(".table-label-columns-move-down").disabled = - state.isBusy || index === rows.length - 1; - }); -} - -function moveTableLabelColumnsRow(state, row, direction) { - if (direction === "up") { - var previous = row.previousElementSibling; - if (previous) { - state.list.insertBefore(row, previous); - } - } else { - var next = row.nextElementSibling; - if (next) { - state.list.insertBefore(next, row); - } - } - updateTableLabelColumnsMoveButtons(state); -} - -function addTableLabelColumnsRow(state, column, checked) { - var row = document.createElement("li"); - row.className = "table-label-columns-row"; - row.innerHTML = - '" + - '
' + - '' + - '' + - "
"; - row.querySelector(".table-label-columns-checkbox").checked = !!checked; - row.querySelector(".table-label-columns-checkbox").value = column; - row.querySelector(".table-label-columns-name").textContent = column; - row - .querySelector(".table-label-columns-move-up") - .addEventListener("click", function () { - moveTableLabelColumnsRow(state, row, "up"); - }); - row - .querySelector(".table-label-columns-move-down") - .addEventListener("click", function () { - moveTableLabelColumnsRow(state, row, "down"); - }); - state.list.appendChild(row); - return row; -} - -function renderTableLabelColumnsFields(state, data) { - state.list.innerHTML = ""; - var current = data.current || []; - var remaining = (data.columns || []).filter(function (column) { - return current.indexOf(column) === -1; - }); - current.forEach(function (column) { - addTableLabelColumnsRow(state, column, true); - }); - remaining.forEach(function (column) { - addTableLabelColumnsRow(state, column, false); - }); - updateTableLabelColumnsMoveButtons(state); -} - -function collectTableLabelColumnsPayload(state) { - var rows = Array.prototype.slice.call( - state.list.querySelectorAll(".table-label-columns-row"), - ); - var columns = []; - rows.forEach(function (row) { - var checkbox = row.querySelector(".table-label-columns-checkbox"); - if (checkbox.checked) { - columns.push(checkbox.value); - } - }); - return columns; -} - -async function submitTableLabelColumns(state, columns) { - if (state.isBusy) { - return; - } - var data = tableLabelColumnsData(); - if (!data || !data.path) { - showTableLabelColumnsDialogError( - state, - "Could not find the set label columns URL.", - ); - return; - } - clearTableLabelColumnsDialogError(state); - setTableLabelColumnsDialogBusy(state, true); - try { - var response = await fetch(data.path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify({ columns: columns }), - }); - var responseData = null; - try { - responseData = await response.json(); - } catch (_error) { - responseData = null; - } - if (!response.ok || (responseData && responseData.ok === false)) { - throw rowMutationRequestError(response, responseData); - } - state.shouldRestoreFocus = false; - state.dialog.close(); - location.reload(); - } catch (error) { - setTableLabelColumnsDialogBusy(state, false); - showTableLabelColumnsDialogError( - state, - error.message || "Could not save label columns", - ); - } -} - -function ensureTableLabelColumnsDialog(manager) { - if (tableLabelColumnsDialogState) { - return tableLabelColumnsDialogState; - } - if (!window.HTMLDialogElement) { - return null; - } - - var dialog = document.createElement("dialog"); - dialog.id = TABLE_LABEL_COLUMNS_DIALOG_ID; - dialog.className = "table-label-columns-dialog"; - dialog.setAttribute("aria-labelledby", "table-label-columns-title"); - dialog.innerHTML = ` -
- Set label column(s) -
- - -

Choose one or more columns to use as the display label for this table's rows. Checked columns are used in the order shown below - use the arrows to reorder them.

-
    -
    - - - -
    - - `; - document.body.appendChild(dialog); - - tableLabelColumnsDialogState = { - dialog: dialog, - form: dialog.querySelector(".table-label-columns-form"), - title: dialog.querySelector(".modal-title"), - error: dialog.querySelector(".table-label-columns-error"), - list: dialog.querySelector(".table-label-columns-list"), - resetButton: dialog.querySelector(".table-label-columns-reset"), - cancelButton: dialog.querySelector(".table-label-columns-cancel"), - saveButton: dialog.querySelector(".table-label-columns-save"), - currentButton: null, - isBusy: false, - isOverridden: false, - shouldRestoreFocus: true, - manager: manager, - }; - - tableLabelColumnsDialogState.form.addEventListener("submit", function (ev) { - ev.preventDefault(); - var state = tableLabelColumnsDialogState; - submitTableLabelColumns(state, collectTableLabelColumnsPayload(state)); - }); - - tableLabelColumnsDialogState.resetButton.addEventListener( - "click", - function () { - submitTableLabelColumns(tableLabelColumnsDialogState, null); - }, - ); - - tableLabelColumnsDialogState.cancelButton.addEventListener( - "click", - function () { - var state = tableLabelColumnsDialogState; - if (!state.isBusy) { - state.shouldRestoreFocus = true; - dialog.close(); - } - }, - ); - - dialog.addEventListener("click", function (ev) { - var state = tableLabelColumnsDialogState; - if (ev.target === dialog && !state.isBusy) { - state.shouldRestoreFocus = true; - dialog.close(); - } - }); - - dialog.addEventListener("keydown", function (ev) { - if (ev.key !== "Escape") { - return; - } - var state = tableLabelColumnsDialogState; - if (state.isBusy) { - ev.preventDefault(); - return; - } - ev.preventDefault(); - state.shouldRestoreFocus = true; - dialog.close(); - }); - - dialog.addEventListener("cancel", function (ev) { - var state = tableLabelColumnsDialogState; - if (state.isBusy) { - ev.preventDefault(); - } else { - state.shouldRestoreFocus = true; - } - }); - - dialog.addEventListener("close", function () { - var state = tableLabelColumnsDialogState; - clearTableLabelColumnsDialogError(state); - setTableLabelColumnsDialogBusy(state, false); - if ( - state.shouldRestoreFocus && - state.currentButton && - document.contains(state.currentButton) - ) { - state.currentButton.focus(); - } - }); - - return tableLabelColumnsDialogState; -} - -function openTableLabelColumnsDialog(button, manager) { - var data = tableLabelColumnsData(); - if (!data) { - return; - } - var state = ensureTableLabelColumnsDialog(manager); - if (!state) { - return; - } - - var menu = button.closest("details"); - if (menu) { - menu.open = false; - } - state.manager = manager; - state.currentButton = button; - state.isOverridden = !!data.isOverridden; - state.shouldRestoreFocus = true; - state.title.textContent = data.tableName - ? "Set label column(s) for " + data.tableName - : "Set label column(s)"; - clearTableLabelColumnsDialogError(state); - renderTableLabelColumnsFields(state, data); - setTableLabelColumnsDialogBusy(state, false); - - if (!state.dialog.open) { - state.dialog.showModal(); - } - var firstCheckbox = state.list.querySelector( - ".table-label-columns-checkbox", - ); - if (firstCheckbox) { - firstCheckbox.focus(); - } -} - -function initTableLabelColumnsActions(manager) { - if (!window.fetch || !window.HTMLDialogElement || !tableLabelColumnsData()) { - return; - } - document.addEventListener("click", function (ev) { - var button = ev.target.closest( - 'button[data-table-action="set-label-columns"]', - ); - if (!button) { - return; - } - ev.preventDefault(); - openTableLabelColumnsDialog(button, manager); - }); -} - document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; @@ -5623,5 +7619,4 @@ document.addEventListener("datasette_init", function (evt) { initRowInsertActions(manager); initRowEditActions(manager); initRowDeleteActions(manager); - initTableLabelColumnsActions(manager); }); diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index a6123daa..f5f977d9 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -62,7 +62,6 @@ def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: "description_html": query.description_html, "hide_sql": query.hide_sql, "fragment": query.fragment, - "params": list(query.parameters), "parameters": list(query.parameters), "is_write": query.is_write, "is_private": query.is_private, @@ -84,7 +83,6 @@ def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: return { "queries": [stored_query_to_dict(query) for query in page.queries], "next": page.next, - "has_more": page.has_more, "limit": page.limit, } diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html index 53a824f1..21a2ea8f 100644 --- a/datasette/templates/_permission_ui_styles.html +++ b/datasette/templates/_permission_ui_styles.html @@ -6,8 +6,20 @@ padding: 1.5em; margin-bottom: 2em; } +.permission-form form { + max-width: 60rem; +} +.permission-form-grid { + display: grid; + gap: 1.5rem; + grid-template-columns: repeat(2, minmax(0, 1fr)); +} +.permission-form-result { + margin-top: 1rem; + max-width: 60rem; +} .form-section { - margin-bottom: 1em; + margin-bottom: 1.25em; } .form-section label { display: block; @@ -15,22 +27,51 @@ font-weight: bold; } .form-section input[type="text"], -.form-section select { - width: 100%; - max-width: 500px; - padding: 0.5em; +.form-section input[type="number"], +.form-section select, +.permission-textarea { + background-color: #fff; + border: 1px solid #aaa; + border-radius: 4px; box-sizing: border-box; - border: 1px solid #ccc; - border-radius: 3px; + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08); + color: #222; + font-family: inherit; + font-size: 1rem; + line-height: 1.4; + max-width: none; + width: 100%; +} +.form-section input[type="text"] { + height: 3rem; + padding: 0.6rem 0.75rem; +} +.form-section input[type="number"] { + height: 3rem; + max-width: 7rem; + padding: 0.6rem 0.75rem; +} +.form-section select { + height: 3rem; + padding: 0.6rem 0.75rem; +} +.permission-textarea { + font-family: monospace; + min-height: 12rem; + padding: 0.75rem; + resize: vertical; } .form-section input[type="text"]:focus, -.form-section select:focus { - outline: 2px solid #0066cc; +.form-section input[type="number"]:focus, +.form-section select:focus, +.permission-textarea:focus { border-color: #0066cc; + box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18); + outline: none; } .form-section small { display: block; - margin-top: 0.3em; + margin-top: 0.45em; color: #666; } .form-actions { @@ -142,4 +183,9 @@ text-align: center; color: #666; } +@media only screen and (max-width: 576px) { + .permission-form-grid { + grid-template-columns: minmax(0, 1fr); + } +} diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html index d7203c1e..8e0f486e 100644 --- a/datasette/templates/_permissions_debug_tabs.html +++ b/datasette/templates/_permissions_debug_tabs.html @@ -44,10 +44,10 @@ diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..fda4032c 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -3,29 +3,11 @@ {% block title %}Debug allow rules{% endblock %} {% block extra_head %} +{% include "_permission_ui_styles.html" %} {% endblock %} @@ -38,24 +20,28 @@ p.message-warning {

    Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

    - -
    -

    - -
    -
    -

    - -
    -
    - -
    - +
    +
    +
    +
    + + +
    +
    + + +
    +
    +
    + +
    + -{% if error %}

    {{ error }}

    {% endif %} + {% if error %}

    {{ error }}

    {% endif %} -{% if result == "True" %}

    Result: allow

    {% endif %} + {% if result == "True" %}

    Result: allow

    {% endif %} -{% if result == "False" %}

    Result: deny

    {% endif %} + {% if result == "False" %}

    Result: deny

    {% endif %} +
    {% endblock %} diff --git a/datasette/templates/debug_actions.html b/datasette/templates/debug_actions.html index 0ef7b329..c9dccaaa 100644 --- a/datasette/templates/debug_actions.html +++ b/datasette/templates/debug_actions.html @@ -9,7 +9,7 @@ {% include "_permissions_debug_tabs.html" %}

    - This Datasette instance has registered {{ data|length }} action{{ data|length != 1 and "s" or "" }}. + This Datasette instance has registered {{ data.actions|length }} action{{ data.actions|length != 1 and "s" or "" }}. Actions are used by the permission system to control access to different features.

    @@ -26,7 +26,7 @@ - {% for action in data %} + {% for action in data.actions %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 4f8106b8..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -49,7 +49,7 @@
    - + Number of results per page (max 200)
    @@ -88,7 +88,7 @@ const hasDebugPermission = {{ 'true' if has_debug_permission else 'false' }}; (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('page'); + const page = params.get('_page'); if (action) { fetchResults(page ? parseInt(page) : 1); } @@ -102,14 +102,14 @@ async function fetchResults(page = 1) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== 'page_size') { + if (value && key !== '_size' && key !== '_page') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('page', page.toString()); - params.append('page_size', pageSize); + params.append('_page', page.toString()); + params.append('_size', pageSize); try { const response = await fetch('{{ urls.path("-/allowed.json") }}?' + params.toString(), { diff --git a/datasette/templates/debug_autocomplete.html b/datasette/templates/debug_autocomplete.html index 1678182e..380639a3 100644 --- a/datasette/templates/debug_autocomplete.html +++ b/datasette/templates/debug_autocomplete.html @@ -26,8 +26,8 @@

    {{ error }}

    {% elif autocomplete_url %}

    {{ database_name }} / {{ table_name }}

    - {% if label_columns %} -

    Label column{{ "s" if label_columns|length > 1 else "" }}: {{ label_columns|join(", ") }}

    + {% if label_column %} +

    Label column: {{ label_column }}

    {% else %}

    No label column detected. Results will use primary key values.

    {% endif %} @@ -64,7 +64,7 @@ - + {% endfor %} diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index 3b229a25..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Permission Check{% endblock %} +{% block title %}Explain a permission decision{% endblock %} {% block extra_head %} @@ -13,29 +13,35 @@ border-radius: 5px; } #output.allowed { - background-color: #e8f5e9; + background-color: #f3fbf4; border: 2px solid #4caf50; } #output.denied { - background-color: #ffebee; + background-color: #fff7f7; border: 2px solid #f44336; } #output h2 { margin-top: 0; } -#output .result-badge { +#output h3 { + margin-bottom: 0.5em; +} +#output .result-badge, +.effect-badge, +.rule-status { display: inline-block; - padding: 0.3em 0.8em; + padding: 0.2em 0.5em; border-radius: 3px; font-weight: bold; - font-size: 1.1em; } -#output .allowed-badge { - background-color: #4caf50; +#output .allowed-badge, +.effect-allow { + background-color: #2e7d32; color: white; } -#output .denied-badge { - background-color: #f44336; +#output .denied-badge, +.effect-deny { + background-color: #c62828; color: white; } .details-section { @@ -48,70 +54,130 @@ .details-section dd { margin-left: 1em; } +.explanation-section { + background: rgba(255, 255, 255, 0.75); + border: 1px solid #ddd; + border-radius: 4px; + margin-top: 1em; + padding: 0 1em 1em; +} +.rules-table { + border-collapse: collapse; + width: 100%; +} +.rules-table th, +.rules-table td { + border-bottom: 1px solid #ddd; + padding: 0.5em; + text-align: left; + vertical-align: top; +} +.rule-status { + background: #e8f5e9; + color: #1b5e20; +} +.rule-ignored { + background: #eee; + color: #555; + font-weight: normal; +} +.requirement-allowed { + color: #1b5e20; +} +.requirement-denied { + color: #b71c1c; +} +@media only screen and (max-width: 576px) { + .rules-table, + .rules-table tbody, + .rules-table tr, + .rules-table td { + display: block; + } + .rules-table thead { + display: none; + } + .rules-table td::before { + content: attr(data-label) ": "; + font-weight: bold; + } +} {% endblock %} {% block content %} -

    Permission check

    +

    Explain a permission decision

    {% set current_tab = "check" %} {% include "_permissions_debug_tabs.html" %} -

    Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

    - -{% if request.actor %} -

    Current actor: {{ request.actor.get("id", "anonymous") }}

    -{% else %} -

    Current actor: anonymous (not logged in)

    -{% endif %} +

    Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

    -
    +
    - + + + Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. +
    + +
    + - The permission action to check + The operation to evaluate
    -
    - +
    + - For database-level permissions, specify the database name + The database or other parent resource
    -
    - - - For table-level permissions, specify the table name (requires parent) +
    + + + The table, query or other child resource
    - +
    +actionSelect.addEventListener('change', updateResourceFields); +(function initializeFromUrl() { + const params = populateFormFromURL(); + updateResourceFields(); + if (params.get('action')) { + performCheck(); + } +})(); + {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 4410a677..8b0cbbcf 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Debug permissions{% endblock %} +{% block title %}Permission activity{% endblock %} {% block extra_head %} {% include "_permission_ui_styles.html" %} @@ -20,60 +20,45 @@ .check-action, .check-when, .check-result { font-size: 1.3em; } -textarea { - height: 10em; - width: 95%; - box-sizing: border-box; - padding: 0.5em; - border: 2px dotted black; -} -.two-col { - display: inline-block; - width: 48%; -} -.two-col label { - width: 48%; -} -@media only screen and (max-width: 576px) { - .two-col { - width: 100%; - } -} {% endblock %} {% block content %} -

    Permission playground

    +

    Permission activity

    {% set current_tab = "permissions" %} {% include "_permissions_debug_tabs.html" %} -

    This tool lets you simulate an actor and a permission check for that actor.

    +

    Raw simulator

    + +

    This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

    -
    -
    - - +
    +
    +
    + + +
    -
    -
    -
    - - -
    -
    - - -
    -
    - - +
    +
    + + +
    +
    + + +
    +
    + + +
    @@ -125,7 +110,7 @@ debugPost.addEventListener('submit', function(ev) { }); -

    Recent permissions checks

    +

    Recent permission checks

    {% if filter != "all" %}All{% else %}All{% endif %}, diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html index aafa755d..233c0e94 100644 --- a/datasette/templates/debug_rules.html +++ b/datasette/templates/debug_rules.html @@ -37,7 +37,7 @@

    - + Number of results per page (max 200)
    @@ -75,7 +75,7 @@ const submitBtn = document.getElementById('submit-btn'); (function() { const params = populateFormFromURL(); const action = params.get('action'); - const page = params.get('page'); + const page = params.get('_page'); if (action) { fetchResults(page ? parseInt(page) : 1); } @@ -89,14 +89,14 @@ async function fetchResults(page = 1) { const params = new URLSearchParams(); for (const [key, value] of formData.entries()) { - if (value && key !== 'page_size') { + if (value && key !== '_size' && key !== '_page') { params.append(key, value); } } const pageSize = document.getElementById('page_size').value || '50'; - params.append('page', page.toString()); - params.append('page_size', pageSize); + params.append('_page', page.toString()); + params.append('_size', pageSize); try { const response = await fetch('{{ urls.path("-/rules.json") }}?' + params.toString(), { diff --git a/datasette/tokens.py b/datasette/tokens.py index 38a55529..4f905339 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -18,6 +18,21 @@ if TYPE_CHECKING: from datasette.app import Datasette +class TokenInvalid(Exception): + """ + Raised by a TokenHandler when a token it recognizes is invalid - + for example a bad signature, malformed payload or expired token. + + Datasette responds to this with an HTTP 401 error. Handlers should + return None instead for tokens they do not recognize at all, so that + other registered handlers get a chance to verify them. + """ + + def __init__(self, message="Invalid token"): + self.message = message + super().__init__(message) + + @dataclasses.dataclass class TokenRestrictions: """ @@ -108,8 +123,12 @@ class TokenHandler: async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: """ - Verify a token and return an actor dict, or None if this handler - does not recognize the token. + Verify a token and return an actor dict. + + Return None if this handler does not recognize the token at all, + so other handlers can try it. Raise TokenInvalid if the token is + recognized but invalid (bad signature, malformed, expired) - the + request will fail with a 401 error. """ raise NotImplementedError @@ -147,29 +166,32 @@ class SignedTokenHandler(TokenHandler): async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: prefix = "dstok_" - if not datasette.setting("allow_signed_tokens"): + if not token.startswith(prefix): + # Not one of our tokens - leave it for other handlers return None + if not datasette.setting("allow_signed_tokens"): + raise TokenInvalid( + "Signed tokens are not enabled for this Datasette instance" + ) + max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl") - if not token.startswith(prefix): - return None - raw = token[len(prefix) :] try: decoded = datasette.unsign(raw, namespace="token") except itsdangerous.BadSignature: - return None + raise TokenInvalid("Invalid token signature") if "t" not in decoded: - return None + raise TokenInvalid("Invalid token: no timestamp") created = decoded["t"] if not isinstance(created, int): - return None + raise TokenInvalid("Invalid token: invalid timestamp") duration = decoded.get("d") if duration is not None and not isinstance(duration, int): - return None + raise TokenInvalid("Invalid token: invalid duration") if (duration is None and max_signed_tokens_ttl) or ( duration is not None @@ -180,7 +202,7 @@ class SignedTokenHandler(TokenHandler): if duration: if time.time() - created > duration: - return None + raise TokenInvalid("Token has expired") actor = {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bd582e22..18d3ba52 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,4 +1,5 @@ import asyncio +import binascii from contextlib import contextmanager import aiofiles import click @@ -236,10 +237,8 @@ class CustomJSONEncoder(json.JSONEncoder): - ``sqlite3.Row`` becomes a tuple - ``sqlite3.Cursor`` becomes a list - If a binary blob can be decoded as UTF-8, the encoder returns it as text. - - If it can't (for example, images), it is encoded as an object, with the actual - data base64-encoded, like so: :: + Binary blobs are encoded as an object, with the actual data base64-encoded, + like so: :: { "$base64": True, @@ -255,17 +254,42 @@ class CustomJSONEncoder(json.JSONEncoder): if isinstance(obj, sqlite3.Cursor): return list(obj) if isinstance(obj, bytes): - # Does it encode to utf8? - try: - return obj.decode("utf8") - except UnicodeDecodeError: - return { - "$base64": True, - "encoded": base64.b64encode(obj).decode("latin1"), - } + return { + "$base64": True, + "encoded": base64.b64encode(obj).decode("latin1"), + } return json.JSONEncoder.default(self, obj) +class WriteJsonValueError(ValueError): + pass + + +def decode_write_json_cell(value): + if not isinstance(value, dict): + return value + keys = set(value.keys()) + if keys == {"$raw"}: + return value["$raw"] + if keys == {"$base64", "encoded"} and value.get("$base64") is True: + encoded = value["encoded"] + if not isinstance(encoded, str): + raise WriteJsonValueError("$base64 encoded value must be a string") + try: + return base64.b64decode(encoded, validate=True) + except binascii.Error as ex: + raise WriteJsonValueError("Invalid $base64 encoded value") from ex + return value + + +def decode_write_json_row(row): + return {key: decode_write_json_cell(value) for key, value in row.items()} + + +def decode_write_json_rows(rows): + return [decode_write_json_row(row) for row in rows] + + @contextmanager def sqlite_timelimit(conn, ms): deadline = time.perf_counter() + (ms / 1000) @@ -612,7 +636,7 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall() fks = [] for info in infos: if info is not None: @@ -1264,10 +1288,20 @@ class StartupError(Exception): pass -_single_line_comment_re = re.compile(r"--.*") -_multi_line_comment_re = re.compile(r"/\*.*?\*/", re.DOTALL) -_single_quote_re = re.compile(r"'(?:''|[^'])*'") -_double_quote_re = re.compile(r'"(?:\"\"|[^"])*"') +# Comments and string literals, matched in a single pass so that whichever +# construct starts first "wins" - this ensures a comment marker inside a string +# literal (or a quote inside a comment) does not confuse the parameter scan. +_comments_and_strings_re = re.compile( + r""" + --[^\n]* # single line comment + | /\*.*?(?:\*/|\Z) # multi line comment, possibly to end-of-input + | '(?:''|[^'])*' # single quoted string ('' escapes a quote) + | "(?:""|[^"])*" # double quoted identifier ("" escapes a quote) + | \[(?:[^\]])*\] # square-bracket quoted identifier + | `(?:``|[^`])*` # backtick quoted identifier + """, + re.DOTALL | re.VERBOSE, +) _named_param_re = re.compile(r":(\w+)") @@ -1278,10 +1312,9 @@ def named_parameters(sql: str) -> List[str]: e.g. for ``select * from foo where id=:id`` this would return ``["id"]`` """ - sql = _single_line_comment_re.sub("", sql) - sql = _multi_line_comment_re.sub("", sql) - sql = _single_quote_re.sub("", sql) - sql = _double_quote_re.sub("", sql) + # Strip comments and string literals first so that any ":name" sequences + # inside them are not mistaken for named parameters + sql = _comments_and_strings_re.sub("", sql) # Extract parameters from what is left return _named_param_re.findall(sql) @@ -1294,6 +1327,54 @@ async def derive_named_parameters(db: "Database", sql: str) -> List[str]: return named_parameters(sql) +def parse_size_limit(value, default, maximum, name="_size"): + """ + Parse a page-size parameter using the same semantics as the table + view's ?_size=: blank means default, "max" means maximum, integers + must be 0 or greater and no larger than maximum. Raises ValueError + with a message suitable for a 400 response. + """ + if value in (None, ""): + return default + if value == "max": + return maximum + try: + size = int(value) + if size < 0: + raise ValueError + except ValueError: + raise ValueError("{} must be a positive integer".format(name)) + if size > maximum: + raise ValueError("{} must be <= {}".format(name, maximum)) + return size + + +UNSTABLE_API_MESSAGE = ( + "This API is not part of Datasette's stable interface and may change at any time" +) + + +def error_body(messages, status): + """ + The canonical JSON error body used by every Datasette JSON error response: + + {"ok": False, "error": "...", "errors": ["...", ...], "status": 400} + + "error" is all of the messages joined with "; ", "errors" is the full + list, "status" matches the HTTP status code. Callers may add extra + context keys to the returned dictionary but must not remove these four. + """ + if isinstance(messages, str): + messages = [messages] + messages = [str(message) for message in messages] + return { + "ok": False, + "error": "; ".join(messages), + "errors": messages, + "status": status, + } + + def add_cors_headers(headers): headers["Access-Control-Allow-Origin"] = "*" headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" @@ -1504,18 +1585,6 @@ _table_config_keys = ( ) -def normalize_label_columns(value): - """ - Normalize a ``label_column`` config value (a string, a list of strings, - or None) to a list of strings. - """ - if value is None: - return [] - if isinstance(value, str): - return [value] - return list(value) - - def move_table_config(metadata: dict, config: dict): """ Move all known table configuration keys from metadata to config. diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index c7137e6b..d767e391 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -252,88 +252,62 @@ async def _build_single_action_sql( ] ) - # Continue with the cascading logic - query_parts.extend( - [ - "child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", + # Continue with the cascading logic. + # Aggregate the RULES by cascade level (small), rather than grouping + # base x rules (which scales with the number of resources). + def _agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,", + " json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons", + f" FROM all_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["child_agg AS ("] + + _agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "parent_agg AS ("] + + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "global_agg AS ("] + + _agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Add anonymous decision logic if needed if include_is_private: - query_parts.extend( - [ - "anon_child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "anon_parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_decisions AS (", - " SELECT", - " b.parent, b.child,", - " CASE", - " WHEN acl.any_deny = 1 THEN 0", - " WHEN acl.any_allow = 1 THEN 1", - " WHEN apl.any_deny = 1 THEN 0", - " WHEN apl.any_allow = 1 THEN 1", - " WHEN agl.any_deny = 1 THEN 0", - " WHEN agl.any_allow = 1 THEN 1", - " ELSE 0", - " END AS anon_is_allowed", - " FROM base b", - " JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))", - " JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))", - " JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))", - "),", + + def _anon_agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow", + f" FROM anon_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["anon_child_agg AS ("] + + _anon_agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "anon_parent_agg AS ("] + + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "anon_global_agg AS ("] + + _anon_agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Final decisions @@ -342,31 +316,28 @@ async def _build_single_action_sql( "decisions AS (", " SELECT", " b.parent, b.child,", - " -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level", + " -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level", " -- Priority order:", - " -- 1. Child-level deny (most specific, blocks access)", - " -- 2. Child-level allow (most specific, grants access)", - " -- 3. Parent-level deny (intermediate, blocks access)", - " -- 4. Parent-level allow (intermediate, grants access)", - " -- 5. Global-level deny (least specific, blocks access)", - " -- 6. Global-level allow (least specific, grants access)", + " -- 1. Child-level deny 2. Child-level allow", + " -- 3. Parent-level deny 4. Parent-level allow", + " -- 5. Global-level deny 6. Global-level allow", " -- 7. Default deny (no rules match)", " CASE", - " WHEN cl.any_deny = 1 THEN 0", - " WHEN cl.any_allow = 1 THEN 1", - " WHEN pl.any_deny = 1 THEN 0", - " WHEN pl.any_allow = 1 THEN 1", - " WHEN gl.any_deny = 1 THEN 0", - " WHEN gl.any_allow = 1 THEN 1", + " WHEN ca.any_deny = 1 THEN 0", + " WHEN ca.any_allow = 1 THEN 1", + " WHEN pa.any_deny = 1 THEN 0", + " WHEN pa.any_allow = 1 THEN 1", + " WHEN ga.any_deny = 1 THEN 0", + " WHEN ga.any_allow = 1 THEN 1", " ELSE 0", " END AS is_allowed,", " CASE", - " WHEN cl.any_deny = 1 THEN cl.deny_reasons", - " WHEN cl.any_allow = 1 THEN cl.allow_reasons", - " WHEN pl.any_deny = 1 THEN pl.deny_reasons", - " WHEN pl.any_allow = 1 THEN pl.allow_reasons", - " WHEN gl.any_deny = 1 THEN gl.deny_reasons", - " WHEN gl.any_allow = 1 THEN gl.allow_reasons", + " WHEN ca.any_deny = 1 THEN ca.deny_reasons", + " WHEN ca.any_allow = 1 THEN ca.allow_reasons", + " WHEN pa.any_deny = 1 THEN pa.deny_reasons", + " WHEN pa.any_allow = 1 THEN pa.allow_reasons", + " WHEN ga.any_deny = 1 THEN ga.deny_reasons", + " WHEN ga.any_allow = 1 THEN ga.allow_reasons", " ELSE '[]'", " END AS reason", ] @@ -374,21 +345,34 @@ async def _build_single_action_sql( if include_is_private: query_parts.append( - " , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private" + " , CASE WHEN (" + "CASE" + " WHEN aca.any_deny = 1 THEN 0" + " WHEN aca.any_allow = 1 THEN 1" + " WHEN apa.any_deny = 1 THEN 0" + " WHEN apa.any_allow = 1 THEN 1" + " WHEN aga.any_deny = 1 THEN 0" + " WHEN aga.any_allow = 1 THEN 1" + " ELSE 0 END" + ") = 0 THEN 1 ELSE 0 END AS is_private" ) query_parts.extend( [ " FROM base b", - " JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))", - " JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))", - " JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))", + " LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child", + " LEFT JOIN parent_agg pa ON pa.parent = b.parent", + " CROSS JOIN global_agg ga", ] ) if include_is_private: - query_parts.append( - " JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))" + query_parts.extend( + [ + " LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child", + " LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent", + " CROSS JOIN anon_global_agg aga", + ] ) query_parts.append(")") @@ -400,8 +384,28 @@ async def _build_single_action_sql( restriction_intersect = "\nINTERSECT\n".join( f"SELECT * FROM ({sql})" for sql in restriction_sqls ) + # Decompose by NULL-pattern so the final filter can use pure-equality + # EXISTS lookups (satisfiable via automatic indexes) instead of a + # correlated OR-scan over the whole list. query_parts.extend( - [",", "restriction_list AS (", f" {restriction_intersect}", ")"] + [ + ",", + "restriction_list AS (", + f" {restriction_intersect}", + "),", + "restriction_exact AS (", + " SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL", + "),", + "restriction_parent_any AS (", + " SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL", + "),", + "restriction_child_any AS (", + " SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL", + "),", + "restriction_all AS (", + " SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1", + ")", + ] ) # Final SELECT @@ -416,10 +420,11 @@ async def _build_single_action_sql( # Add restriction filter if there are restrictions if restriction_sqls: query_parts.append(""" - AND EXISTS ( - SELECT 1 FROM restriction_list r - WHERE (r.parent = decisions.parent OR r.parent IS NULL) - AND (r.child = decisions.child OR r.child IS NULL) + AND ( + EXISTS (SELECT 1 FROM restriction_all) + OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent) + OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child) + OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child) )""") # Add parent filter if specified @@ -673,3 +678,239 @@ async def check_permission_for_resource( child=child, ) return results[action] + + +async def explain_permission_for_resource( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Explain a permission decision for one action and resource. + + This is intended for Datasette's permission debugging tools. It uses the + same ``permission_resources_sql`` hook results and the same resolution + rules as :func:`check_permissions_for_actions`, but also returns the + matching rules, actor restriction results and ``also_requires`` chain. + + The returned dictionary is part of Datasette's unstable debugging API. + """ + + action_obj = datasette.actions.get(action) + if action_obj is None: + raise ValueError(f"Unknown action: {action}") + + explanation = await _explain_single_action( + datasette=datasette, + actor=actor, + action=action, + parent=parent, + child=child, + ) + + required_actions = [] + if action_obj.also_requires: + required = await explain_permission_for_resource( + datasette=datasette, + actor=actor, + action=action_obj.also_requires, + parent=parent, + child=child, + ) + required_actions.append(required) + + explanation["required_actions"] = required_actions + explanation["allowed"] = bool( + explanation["rule_allowed"] + and explanation["restriction_allowed"] + and all(required["allowed"] for required in required_actions) + ) + explanation["summary"] = _permission_explanation_summary(explanation) + return explanation + + +async def _explain_single_action( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Return matching rules and restrictions for a single action.""" + from datasette.utils.permissions import SKIP_PERMISSION_CHECKS + + permission_sqls = await gather_permission_sql_from_hooks( + datasette=datasette, + actor=actor, + action=action, + ) + + if permission_sqls is SKIP_PERMISSION_CHECKS: + return { + "action": action, + "rule_allowed": True, + "restriction_allowed": True, + "winning_scope": "global", + "matched_rules": [ + { + "scope": "global", + "effect": "allow", + "source": "skip_permission_checks", + "reason": "Permission checks were explicitly skipped", + "decisive": True, + "ignored_because": None, + } + ], + "restrictions": [], + } + + db = datasette.get_internal_database() + matched_rules = [] + restrictions = [] + + for permission_sql in permission_sqls: + params = dict(permission_sql.params or {}) + parent_param = _unused_parameter_name(params, "_explain_parent") + params[parent_param] = parent + child_param = _unused_parameter_name(params, "_explain_child") + params[child_param] = child + + if permission_sql.sql: + rows = await db.execute( + f""" + SELECT parent, child, allow, reason + FROM ({permission_sql.sql}) AS permission_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child = :{child_param}) + """, + params, + ) + for row in rows: + specificity = ( + 2 + if row["child"] is not None + else 1 if row["parent"] is not None else 0 + ) + matched_rules.append( + { + "scope": ("resource", "parent", "global")[2 - specificity], + "effect": "allow" if row["allow"] else "deny", + "source": permission_sql.source, + "reason": row["reason"], + "_specificity": specificity, + } + ) + + if permission_sql.restriction_sql: + restriction_row = ( + await db.execute( + f""" + SELECT EXISTS( + SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child = :{child_param}) + ) AS resource_is_in_allowlist + """, + params, + ) + ).first() + restriction_allowed = bool(restriction_row[0]) + restrictions.append( + { + "source": permission_sql.source, + "allowed": restriction_allowed, + "reason": params.get("deny") + or ( + "Resource is included in this restriction allowlist" + if restriction_allowed + else "Resource is not included in this restriction allowlist" + ), + } + ) + + matched_rules.sort( + key=lambda rule: ( + -rule["_specificity"], + 0 if rule["effect"] == "deny" else 1, + rule["source"] or "", + rule["reason"] or "", + ) + ) + + if matched_rules: + winning_specificity = matched_rules[0]["_specificity"] + winning_rules = [ + rule + for rule in matched_rules + if rule["_specificity"] == winning_specificity + ] + rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules) + winning_scope = winning_rules[0]["scope"] + else: + winning_specificity = None + rule_allowed = False + winning_scope = None + + for rule in matched_rules: + specificity = rule.pop("_specificity") + if specificity != winning_specificity: + rule["decisive"] = False + rule["ignored_because"] = "A more specific rule matched" + elif not rule_allowed and rule["effect"] == "allow": + rule["decisive"] = False + rule["ignored_because"] = "A deny rule matched at the same scope" + else: + rule["decisive"] = True + rule["ignored_because"] = None + + return { + "action": action, + "rule_allowed": rule_allowed, + "restriction_allowed": all( + restriction["allowed"] for restriction in restrictions + ), + "winning_scope": winning_scope, + "matched_rules": matched_rules, + "restrictions": restrictions, + } + + +def _unused_parameter_name(params: dict, preferred: str) -> str: + """Return a SQL parameter name that is not already in ``params``.""" + candidate = preferred + suffix = 2 + while candidate in params: + candidate = f"{preferred}_{suffix}" + suffix += 1 + return candidate + + +def _permission_explanation_summary(explanation: dict) -> str: + denied_requirement = next( + ( + required + for required in explanation["required_actions"] + if not required["allowed"] + ), + None, + ) + if denied_requirement: + return ( + f"Denied because {explanation['action']} also requires " + f"{denied_requirement['action']}, which was denied." + ) + if not explanation["matched_rules"]: + return "Denied because no permission rule matched this actor and resource." + if not explanation["rule_allowed"]: + return ( + f"Denied by a {explanation['winning_scope']}-level rule. " + "Deny rules take precedence over allow rules at the same scope." + ) + if not explanation["restriction_allowed"]: + return ( + "Denied because the resource is not included in the actor's restrictions." + ) + return f"Allowed by the matching {explanation['winning_scope']}-level rule." diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index e1631b10..610b86f2 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,6 +1,6 @@ import json from typing import Optional -from datasette.utils import MultiParams, calculate_etag, sha256_file +from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file from datasette.utils.multipart import ( parse_form_data, MultipartParseError, @@ -67,13 +67,25 @@ class BadRequest(Base400): status = 400 +class PayloadTooLarge(Base400): + status = 413 + + SAMESITE_VALUES = ("strict", "lax", "none") +# Bodies read fully into memory (post_body/post_vars/json) are capped at this +# size unless the max_post_body_bytes setting says otherwise. Kept deliberately +# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk, +# while these bodies are held in RAM and json.loads() can multiply their +# footprint several times over. +DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB + class Request: - def __init__(self, scope, receive): + def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES): self.scope = scope self.receive = receive + self.max_post_body_bytes = max_post_body_bytes def __repr__(self): return ''.format(self.method, self.url) @@ -141,15 +153,43 @@ class Request: def actor(self): return self.scope.get("actor", None) - async def post_body(self): - body = b"" + async def post_body(self, max_bytes=None): + """ + Read the request body fully into memory. + + The body is capped at max_bytes - or self.max_post_body_bytes + (default 2MB, set from the max_post_body_bytes setting for requests + created by Datasette) if max_bytes is not provided. Pass max_bytes=0 + to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded - + oversized bodies are rejected as soon as the limit is passed, without + buffering the rest. + """ + if max_bytes is None: + max_bytes = self.max_post_body_bytes + too_large = PayloadTooLarge( + "Request body exceeded maximum size of {} bytes".format(max_bytes) + ) + if max_bytes: + # Reject early if the client declares an oversized body + try: + if int(self.headers.get("content-length", "")) > max_bytes: + raise too_large + except ValueError: + # Missing or malformed - the streaming check below still applies + pass + chunks = [] + received = 0 more_body = True while more_body: message = await self.receive() assert message["type"] == "http.request", message - body += message.get("body", b"") + chunk = message.get("body", b"") + received += len(chunk) + if max_bytes and received > max_bytes: + raise too_large + chunks.append(chunk) more_body = message.get("more_body", False) - return body + return b"".join(chunks) async def post_vars(self): body = await self.post_body() @@ -535,6 +575,18 @@ class Response: content_type="application/json; charset=utf-8", ) + @classmethod + def error(cls, messages, status=400, headers=None): + """ + A JSON error response using Datasette's standard error format. + + messages can be a single string or a list of strings. For errors + that should content-negotiate between JSON and HTML, raise + Forbidden, NotFound, BadRequest or DatasetteError instead and let + Datasette's error handling hooks build the response. + """ + return cls.json(error_body(messages, status), status=status, headers=headers) + @classmethod def redirect(cls, path, status=302, headers=None): headers = headers or {} diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index a629f0fe..702b53d8 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,9 +1,30 @@ import textwrap -from datasette.utils import table_column_details +from sqlite_utils import Database as SQLiteUtilsDatabase +from sqlite_utils import Migrations -async def init_internal_db(db): - create_tables_sql = textwrap.dedent(""" +from datasette.utils import escape_sqlite, table_column_details + +INTERNAL_DB_SCHEMA_TABLES = { + "catalog_databases", + "catalog_tables", + "catalog_views", + "catalog_columns", + "catalog_indexes", + "catalog_foreign_keys", + "metadata_instance", + "metadata_databases", + "metadata_resources", + "metadata_columns", + "column_types", + "queries", +} + +INTERNAL_DB_SCHEMA_INDEXES = { + "queries_owner_idx", +} + +INTERNAL_DB_SCHEMA_SQL = textwrap.dedent(""" CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, @@ -67,106 +88,101 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name) ); + + CREATE TABLE IF NOT EXISTS metadata_instance ( + key text, + value text, + unique(key) + ); + + CREATE TABLE IF NOT EXISTS metadata_databases ( + database_name text, + key text, + value text, + unique(database_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_resources ( + database_name text, + resource_name text, + key text, + value text, + unique(database_name, resource_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_columns ( + database_name text, + resource_name text, + column_name text, + key text, + value text, + unique(database_name, resource_name, column_name, key) + ); + + CREATE TABLE IF NOT EXISTS column_types ( + database_name TEXT NOT NULL, + resource_name TEXT NOT NULL, + column_name TEXT NOT NULL, + column_type TEXT NOT NULL, + config TEXT, + PRIMARY KEY (database_name, resource_name, column_name) + ); + + CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name) + ); + + CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); """).strip() - await db.execute_write_script(create_tables_sql) - await initialize_metadata_tables(db) -async def initialize_metadata_tables(db): - await db.execute_write_script(textwrap.dedent(""" - CREATE TABLE IF NOT EXISTS metadata_instance ( - key text, - value text, - unique(key) - ); - - CREATE TABLE IF NOT EXISTS metadata_databases ( - database_name text, - key text, - value text, - unique(database_name, key) - ); - - CREATE TABLE IF NOT EXISTS metadata_resources ( - database_name text, - resource_name text, - key text, - value text, - unique(database_name, resource_name, key) - ); - - CREATE TABLE IF NOT EXISTS metadata_columns ( - database_name text, - resource_name text, - column_name text, - key text, - value text, - unique(database_name, resource_name, column_name, key) - ); - - CREATE TABLE IF NOT EXISTS column_types ( - database_name TEXT NOT NULL, - resource_name TEXT NOT NULL, - column_name TEXT NOT NULL, - column_type TEXT NOT NULL, - config TEXT, - PRIMARY KEY (database_name, resource_name, column_name) - ); - - CREATE TABLE IF NOT EXISTS label_columns ( - database_name TEXT NOT NULL, - resource_name TEXT NOT NULL, - columns TEXT NOT NULL, -- JSON list of column names, order = label order - PRIMARY KEY (database_name, resource_name) - ); - - CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) - ); - - CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); - """)) +internal_migrations = Migrations("datasette_internal") -async def populate_schema_tables(internal_db, db): +def _internal_schema_exists(db): + table_names = set(db.table_names()) + if not INTERNAL_DB_SCHEMA_TABLES.issubset(table_names): + return False + index_names = { + row[0] + for row in db.execute("select name from sqlite_master where type = 'index'") + } + return INTERNAL_DB_SCHEMA_INDEXES.issubset(index_names) + + +@internal_migrations(name="0001_initial") +def initial_internal_schema(db): + if _internal_schema_exists(db): + return + db.executescript(INTERNAL_DB_SCHEMA_SQL) + + +async def init_internal_db(db): + def apply_migrations(conn): + internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False)) + + await db.execute_write_fn(apply_migrations, transaction=False) + + +async def populate_schema_tables(internal_db, db, schema_version): database_name = db.name - def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_views WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) - - await internal_db.execute_write_fn(delete_everything) - tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows @@ -197,7 +213,7 @@ async def populate_schema_tables(internal_db, db): for column in columns ) foreign_keys = conn.execute( - f"PRAGMA foreign_key_list([{table_name}])" + f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" ).fetchall() foreign_keys_to_insert.extend( { @@ -206,7 +222,9 @@ async def populate_schema_tables(internal_db, db): } for foreign_key in foreign_keys ) - indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() + indexes = conn.execute( + f"PRAGMA index_list({escape_sqlite(table_name)})" + ).fetchall() indexes_to_insert.extend( { **{"database_name": database_name, "table_name": table_name}, @@ -230,47 +248,76 @@ async def populate_schema_tables(internal_db, db): indexes_to_insert, ) = await db.execute_fn(collect_info) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - tables_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_views (database_name, view_name, rootpage, sql) - values (?, ?, ?, ?) - """, - views_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + def replace_catalog(conn): + # Delete child rows before their catalog_tables parents so this also + # works if a prepare_connection plugin enables foreign key enforcement. + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + ): + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [database_name], + ) + conn.execute( + """ + INSERT OR REPLACE INTO catalog_databases ( + database_name, path, is_memory, schema_version + ) VALUES (?, ?, ?, ?) + """, + [ + database_name, + str(db.path) if db.path is not None else None, + db.is_memory, + schema_version, + ], ) - """, - columns_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + conn.executemany( + """ + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, ) - """, - foreign_keys_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial + conn.executemany( + """ + INSERT INTO catalog_views (database_name, view_name, rootpage, sql) + values (?, ?, ?, ?) + """, + views_to_insert, ) - """, - indexes_to_insert, - ) + conn.executemany( + """ + INSERT INTO catalog_columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + columns_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + foreign_keys_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + indexes_to_insert, + ) + + await internal_db.execute_write_fn(replace_catalog) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 3a509bd2..1be28982 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -488,17 +488,17 @@ def analyze_sql_tables( and key.operation in {"create", "alter", "drop"} for key in operations ) - dropped_tables = { + dropped_tables_and_views = { (key.database, key.table) for key in operations - if key.operation == "drop" and key.target_type == "table" + if key.operation == "drop" and key.target_type in {"table", "view"} } def key_is_drop_table_delete(key: OperationKey) -> bool: return ( key.operation == "delete" and key.target_type == "table" - and (key.database, key.table) in dropped_tables + and (key.database, key.table) in dropped_tables_and_views ) has_user_table_access_in_schema_operation = any( diff --git a/datasette/version.py b/datasette/version.py index 49d270e4..8e238ab5 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a35" +__version__ = "1.0a37" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/base.py b/datasette/views/base.py index 30026f4b..66e14a6d 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -28,12 +28,15 @@ class DatasetteError(Exception): status=500, template=None, message_is_html=False, + plain_message=None, ): self.message = message self.title = title self.error_dict = error_dict or {} self.status = status self.message_is_html = message_is_html + # Plain text used for JSON error responses when message is HTML + self.plain_message = plain_message class View: @@ -49,9 +52,7 @@ class View: request.path.endswith(".json") or request.headers.get("content-type") == "application/json" ): - response = Response.json( - {"ok": False, "error": "Method not allowed"}, status=405 - ) + response = Response.error("Method not allowed", 405) else: response = Response.text("Method not allowed", status=405) return response @@ -90,9 +91,7 @@ class BaseView: request.path.endswith(".json") or request.headers.get("content-type") == "application/json" ): - response = Response.json( - {"ok": False, "error": "Method not allowed"}, status=405 - ) + response = Response.error("Method not allowed", 405) else: response = Response.text("Method not allowed", status=405) return response @@ -180,10 +179,6 @@ class BaseView: return view -def _error(messages, status=400): - return Response.json({"ok": False, "errors": messages}, status=status) - - async def stream_csv(datasette, fetch_data, request, database): kwargs = {} stream = request.args.get("_stream") diff --git a/datasette/views/database.py b/datasette/views/database.py index e02de657..11646f45 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -8,7 +8,7 @@ import markupsafe import os import textwrap -from datasette.extras import extra_names_from_request +from datasette.extras import extra_names_from_request, ExtraScope from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, stored_query_to_dict @@ -16,6 +16,7 @@ from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, await_me_maybe, + error_body, call_with_supported_arguments, named_parameters as derive_named_parameters, format_bytes, @@ -325,7 +326,7 @@ class DatabaseContext(Context): database_color: str = field(metadata={"help": "The color assigned to the database"}) database_page_data: dict = field( metadata={ - "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.' + "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.' } ) database_actions: callable = field( @@ -607,11 +608,7 @@ class QueryView(View): "_json" ): return Response.json( - { - "ok": False, - "message": ex.message, - "redirect": None, - }, + dict(error_body([ex.message], 403), redirect=None), status=403, ) datasette.add_message(request, ex.message, datasette.ERROR) @@ -646,8 +643,15 @@ class QueryView(View): ok = None redirect_url = None try: + execute_write_kwargs = {"request": request} + if stored_query.is_trusted: + analysis = await db.analyze_sql(stored_query.sql, params_for_query) + if any( + operation.operation == "vacuum" for operation in analysis.operations + ): + execute_write_kwargs["transaction"] = False cursor = await db.execute_write( - stored_query.sql, params_for_query, request=request + stored_query.sql, params_for_query, **execute_write_kwargs ) # success message can come from on_success_message or on_success_message_sql message = None @@ -681,12 +685,17 @@ class QueryView(View): redirect_url = stored_query.on_error_redirect ok = False if should_return_json: + if ok: + return Response.json( + { + "ok": True, + "message": message, + "redirect": redirect_url, + } + ) return Response.json( - { - "ok": ok, - "message": message, - "redirect": redirect_url, - } + dict(error_body([message], 400), redirect=redirect_url), + status=400, ) else: datasette.add_message(request, message, message_type) @@ -817,6 +826,10 @@ class QueryView(View): title="SQL Interrupted", status=400, message_is_html=True, + plain_message=( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ), ) except sqlite3.DatabaseError as ex: query_error = str(ex) @@ -849,8 +862,11 @@ class QueryView(View): return await stream_csv(datasette, fetch_data_for_csv, request, db.name) elif format_ in datasette.renderers.keys(): + if not sql: + raise DatasetteError("?sql= is required", status=400) data = {"ok": True, "rows": rows, "columns": columns} extras = extra_names_from_request(request) + table_extra_registry.validate_requested(extras, ExtraScope.QUERY) if extras: query_extra_context = QueryExtraContext( datasette=datasette, diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index b7e8288e..dd35b127 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -2,10 +2,10 @@ import re from urllib.parse import urlencode from datasette.resources import DatabaseResource -from datasette.utils import sqlite3 +from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3 from datasette.utils.asgi import Response -from .base import BaseView, _error +from .base import BaseView from .database import display_rows as display_query_rows from .query_helpers import ( QueryValidationError, @@ -348,7 +348,7 @@ class ExecuteWriteView(BaseView): ) if not db.is_mutable: return _block_framing( - _error( + Response.error( ["Cannot execute write SQL because this database is immutable."], 403, ) @@ -367,10 +367,10 @@ class ExecuteWriteView(BaseView): actor=request.actor, ): return _block_framing( - _error(["Permission denied: need execute-write-sql"], 403) + Response.error(["Permission denied: need execute-write-sql"], 403) ) if not db.is_mutable: - return _block_framing(_error(["Database is immutable"], 403)) + return _block_framing(Response.error(["Database is immutable"], 403)) data = {} is_json = request.headers.get("content-type", "").startswith("application/json") @@ -384,7 +384,7 @@ class ExecuteWriteView(BaseView): ) except QueryValidationError as ex: if _wants_json(request, is_json, data): - return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.error([ex.message], ex.status)) if ex.flash: self.ds.add_message(request, ex.message, self.ds.ERROR) return await self._render_form( @@ -405,7 +405,7 @@ class ExecuteWriteView(BaseView): except sqlite3.DatabaseError as ex: message = str(ex) if wants_json: - return _block_framing(_error([message], 400)) + return _block_framing(Response.error([message], 400)) return await self._render_form( request, db, @@ -488,20 +488,18 @@ class ExecuteWriteAnalyzeView(BaseView): actor=request.actor, ): return _block_framing( - _error(["Permission denied: need execute-write-sql"], 403) + Response.error(["Permission denied: need execute-write-sql"], 403) ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - _error( + Response.error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) ) sql = request.args.get("sql") or "" - return _block_framing( - Response.json( - await _execute_write_analysis_data(self.ds, db, sql, request.actor) - ) - ) + analysis = await _execute_write_analysis_data(self.ds, db, sql, request.actor) + analysis["unstable"] = UNSTABLE_API_MESSAGE + return _block_framing(Response.json(analysis)) diff --git a/datasette/views/index.py b/datasette/views/index.py index 6a9462ac..67296cd1 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -6,6 +6,7 @@ from datasette.utils import ( await_me_maybe, make_slot_function, CustomJSONEncoder, + UNSTABLE_API_MESSAGE, ) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -151,7 +152,9 @@ class IndexView(BaseView): return Response( json.dumps( { - "databases": {db["name"]: db for db in databases}, + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "databases": databases, "metadata": await self.ds.get_instance_metadata(), }, cls=CustomJSONEncoder, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 026a999f..588891d4 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -13,6 +13,7 @@ from datasette.write_sql import ( operation_is_write, ) from datasette.utils import ( + parse_size_limit, named_parameters as derive_named_parameters, escape_sqlite, path_from_row_pks, @@ -32,7 +33,6 @@ _query_fields = { "hide_sql", "fragment", "parameters", - "params", "is_private", "on_success_message", "on_success_redirect", @@ -94,13 +94,11 @@ def _as_optional_bool(value, name): raise QueryValidationError("{} must be 0 or 1".format(name)) -def _query_list_limit(value, default=50): - if value in (None, ""): - return default +def _query_list_limit(value, default, maximum): try: - return min(max(1, int(value)), 1000) + return parse_size_limit(value, default, maximum) except ValueError as ex: - raise QueryValidationError("_size must be an integer") from ex + raise QueryValidationError(str(ex)) from ex def _derived_query_parameters(sql): @@ -541,7 +539,7 @@ async def _prepare_query_create(datasette, request, db, data): raise QueryValidationError("Writable query fields require writable SQL") parameters = _coerce_query_parameters( - data.get("parameters", data.get("params")), + data.get("parameters"), derived, ) return { @@ -586,9 +584,9 @@ async def _prepare_query_update(datasette, request, db, existing: StoredQuery, u actor=request.actor, ) - if "parameters" in update or "params" in update: + if "parameters" in update: parameters = _coerce_query_parameters( - update.get("parameters", update.get("params")), + update.get("parameters"), derived, ) elif "sql" in update: diff --git a/datasette/views/row.py b/datasette/views/row.py index e93f60e2..c90a3bbe 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,25 +8,27 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils -from datasette.utils.asgi import NotFound, Forbidden, Response +from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource -from .base import BaseView, DatasetteError, _error, stream_csv +from .base import BaseView, DatasetteError, stream_csv from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, CustomRow, + decode_write_json_row, InvalidSql, make_slot_function, path_from_row_pks, - path_with_added_args, path_with_format, path_with_removed_args, to_css_class, escape_sqlite, sqlite3, + WriteJsonValueError, ) from datasette.plugins import pm from datasette.extras import extra_names_from_request, ExtraScope @@ -34,7 +36,7 @@ from . import Context, from_extra from .table import ( display_columns_and_rows, _table_page_data, - row_label_from_label_columns, + row_label_from_label_column, ) from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -198,6 +200,10 @@ class RowView(BaseView): title="SQL Interrupted", status=400, message_is_html=True, + plain_message=( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ), ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) @@ -209,18 +215,6 @@ class RowView(BaseView): end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - # Special case for .jsono extension - redirect to _shape=objects - if format_ == "jsono": - return self.redirect( - request, - path_with_added_args( - request, - {"_shape": "objects"}, - path=request.path.rsplit(".jsono", 1)[0] + ".json", - ), - forward_querystring=False, - ) - if format_ in self.ds.renderers.keys(): # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) @@ -495,12 +489,10 @@ class RowView(BaseView): "{}".format(cell["value"]) ) - label_columns = ( - await db.label_columns_for_table(table) if is_table else None - ) + label_column = await db.label_column_for_table(table) if is_table else None row_path = path_from_row_pks(rows[0], pks, False) pk_path = path_from_row_pks(rows[0], pks, False, False) - row_label = row_label_from_label_columns(expanded_rows[0], label_columns) + row_label = row_label_from_label_column(expanded_rows[0], label_column) for display_row in display_rows: display_row.pk_path = pk_path display_row.row_path = row_path @@ -587,7 +579,6 @@ class RowView(BaseView): is_view=not is_table, table_insert_ui=None, table_alter_ui=None, - label_columns_ui=None, ), "row_actions": row_actions, "top_row": make_slot_function( @@ -612,6 +603,9 @@ class RowView(BaseView): } extras = extra_names_from_request(request) + if request.url_vars.get("format"): + # Data formats reject unknown extras; HTML ignores them + table_extra_registry.validate_requested(extras, ExtraScope.ROW) # Process extras row_extra_context = RowExtraContext( @@ -706,8 +700,8 @@ def _truncated_row_flash_label(label): async def _row_flash_message(db, action, resolved, row=None): pk_label = ", ".join(resolved.pk_values) - label_columns = await db.label_columns_for_table(resolved.table) - label = row_label_from_label_columns(row or resolved.row, label_columns) + label_column = await db.label_column_for_table(resolved.table) + label = row_label_from_label_column(row or resolved.row, label_column) if label: label = _truncated_row_flash_label(label) if label and label != pk_label: @@ -721,11 +715,13 @@ async def _resolve_row_and_check_permission(datasette, request, permission): try: resolved = await datasette.resolve_row(request) except DatabaseNotFound as e: - return False, _error(["Database not found: {}".format(e.database_name)], 404) + return False, Response.error( + ["Database not found: {}".format(e.database_name)], 404 + ) except TableNotFound as e: - return False, _error(["Table not found: {}".format(e.table)], 404) + return False, Response.error(["Table not found: {}".format(e.table)], 404) except RowNotFound as e: - return False, _error(["Record not found: {}".format(e.pk_values)], 404) + return False, Response.error(["Record not found: {}".format(e.pk_values)], 404) # Ensure user has permission to delete this row if not await datasette.allowed( @@ -733,7 +729,7 @@ async def _resolve_row_and_check_permission(datasette, request, permission): resource=TableResource(database=resolved.db.name, table=resolved.table), actor=request.actor, ): - return False, _error(["Permission denied"], 403) + return False, Response.error(["Permission denied"], 403) return True, resolved @@ -758,7 +754,7 @@ class RowDeleteView(BaseView): try: await resolved.db.execute_write_fn(delete_row, request=request) except Exception as e: - return _error([str(e)], 500) + return Response.error([str(e)], 400) await self.ds.track_event( DeleteRowEvent( @@ -797,18 +793,24 @@ class RowUpdateView(BaseView): try: data = await request.json() except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)]) + return Response.error(["Invalid JSON: {}".format(e)]) + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if not isinstance(data, dict): - return _error(["JSON must be a dictionary"]) + return Response.error(["JSON must be a dictionary"]) if "update" not in data or not isinstance(data["update"], dict): - return _error(["JSON must contain an update dictionary"]) + return Response.error(["JSON must contain an update dictionary"]) invalid_keys = set(data.keys()) - {"update", "return", "alter"} if invalid_keys: - return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + return Response.error(["Invalid keys: {}".format(", ".join(invalid_keys))]) update = data["update"] + try: + update = decode_write_json_row(update) + except WriteJsonValueError as e: + return Response.error([str(e)], 400) # Validate column types from datasette.views.table import _validate_column_types @@ -817,7 +819,7 @@ class RowUpdateView(BaseView): self.ds, resolved.db.name, resolved.table, [update] ) if ct_errors: - return _error(ct_errors, 400) + return Response.error(ct_errors, 400) alter = data.get("alter") if alter and not await self.ds.allowed( @@ -825,7 +827,7 @@ class RowUpdateView(BaseView): resource=TableResource(database=resolved.db.name, table=resolved.table), actor=request.actor, ): - return _error(["Permission denied for alter-table"], 403) + return Response.error(["Permission denied for alter-table"], 403) def update_row(conn): sqlite_utils.Database(conn)[resolved.table].update( @@ -835,7 +837,7 @@ class RowUpdateView(BaseView): try: await resolved.db.execute_write_fn(update_row, request=request) except Exception as e: - return _error([str(e)], 400) + return Response.error([str(e)], 400) result = {"ok": True} returned_row = None @@ -844,7 +846,7 @@ class RowUpdateView(BaseView): resolved.sql, resolved.params, truncate=True ) returned_row = results.dicts()[0] - result["row"] = returned_row + result["rows"] = [returned_row] await self.ds.track_event( UpdateRowEvent( @@ -870,4 +872,4 @@ class RowUpdateView(BaseView): self.ds.INFO, ) - return Response.json(result, status=200) + return Response.json(result, status=200, default=CustomJSONEncoder().default) diff --git a/datasette/views/special.py b/datasette/views/special.py index 824821c8..28d34208 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -6,9 +6,12 @@ from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( + UNSTABLE_API_MESSAGE, actor_matches_allow, + parse_size_limit, add_cors_headers, await_me_maybe, + error_body, tilde_encode, tilde_decode, ) @@ -52,9 +55,9 @@ class JsonDataView(BaseView): if self.permission: await self.ds.ensure_permission(action=self.permission, actor=request.actor) if self.needs_request: - data = self.data_callback(request) + data = await await_me_maybe(self.data_callback(request)) else: - data = self.data_callback() + data = await await_me_maybe(self.data_callback()) # Return JSON or HTML depending on format parameter as_format = request.url_vars.get("format") @@ -62,6 +65,8 @@ class JsonDataView(BaseView): headers = {} if self.ds.cors: add_cors_headers(headers) + if isinstance(data, dict): + data = {"ok": True, **data} return Response.json(data, headers=headers) else: context = { @@ -121,13 +126,13 @@ class AutocompleteDebugView(BaseView): if scanned >= 100: break continue - label_columns = await db.label_columns_for_table(table_name) - if label_columns: + label_column = await db.label_column_for_table(table_name) + if label_column: suggestions.append( { "database": database_name, "table": table_name, - "label_columns": label_columns, + "label_column": label_column, "url": self.ds.urls.path( "-/debug/autocomplete?" + urllib.parse.urlencode( @@ -177,9 +182,7 @@ class AutocompleteDebugView(BaseView): "autocomplete_url": "{}/-/autocomplete".format( self.ds.urls.table(database_name, table_name) ), - "label_columns": await db.label_columns_for_table( - table_name - ), + "label_column": await db.label_column_for_table(table_name), } ) else: @@ -294,6 +297,12 @@ class PermissionsDebugView(BaseView): response, status = await _check_permission_for_actor( self.ds, permission, parent, child, actor ) + if response.get("ok"): + response = { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + **response, + } return Response.json(response, status=status) @@ -350,29 +359,32 @@ class AllowedResourcesView(BaseView): async def _allowed_payload(self, request, has_debug_permission): action = request.args.get("action") if not action: - return {"error": "action parameter is required"}, 400 + return error_body("action parameter is required", 400), 400 if action not in self.ds.actions: - return {"error": f"Unknown action: {action}"}, 404 + return error_body(f"Unknown action: {action}", 404), 404 actor = request.actor if isinstance(request.actor, dict) else None actor_id = actor.get("id") if actor else None parent_filter = request.args.get("parent") child_filter = request.args.get("child") if child_filter and not parent_filter: - return {"error": "parent must be provided when child is specified"}, 400 + return ( + error_body("parent must be provided when child is specified", 400), + 400, + ) try: - page = int(request.args.get("page", "1")) - page_size = int(request.args.get("page_size", "50")) + page = int(request.args.get("_page", "1")) + if page < 1: + raise ValueError except ValueError: - return {"error": "page and page_size must be integers"}, 400 - if page < 1: - return {"error": "page must be >= 1"}, 400 - if page_size < 1: - return {"error": "page_size must be >= 1"}, 400 - max_page_size = 200 - if page_size > max_page_size: - page_size = max_page_size + return error_body("_page must be a positive integer", 400), 400 + try: + page_size = parse_size_limit( + request.args.get("_size"), default=50, maximum=200 + ) + except ValueError as ex: + return error_body(str(ex), 400), 400 offset = (page - 1) * page_size # Use the simplified allowed_resources method @@ -412,6 +424,7 @@ class AllowedResourcesView(BaseView): # If catalog tables don't exist yet, return empty results return ( { + "ok": True, "action": action, "actor_id": actor_id, "page": page, @@ -436,16 +449,17 @@ class AllowedResourcesView(BaseView): def build_page_url(page_number): pairs = [] for key in request.args: - if key in {"page", "page_size"}: + if key in {"_page", "_size"}: continue for value in request.args.getlist(key): pairs.append((key, value)) - pairs.append(("page", str(page_number))) - pairs.append(("page_size", str(page_size))) + pairs.append(("_page", str(page_number))) + pairs.append(("_size", str(page_size))) query = urllib.parse.urlencode(pairs) return f"{request.path}?{query}" response = { + "ok": True, "action": action, "actor_id": actor_id, "page": page, @@ -487,26 +501,24 @@ class PermissionRulesView(BaseView): # JSON API - action parameter is required action = request.args.get("action") if not action: - return Response.json({"error": "action parameter is required"}, status=400) + return Response.error("action parameter is required", 400) if action not in self.ds.actions: - return Response.json({"error": f"Unknown action: {action}"}, status=404) + return Response.error(f"Unknown action: {action}", 404) actor = request.actor if isinstance(request.actor, dict) else None try: - page = int(request.args.get("page", "1")) - page_size = int(request.args.get("page_size", "50")) + page = int(request.args.get("_page", "1")) + if page < 1: + raise ValueError except ValueError: - return Response.json( - {"error": "page and page_size must be integers"}, status=400 + return Response.error("_page must be a positive integer", 400) + try: + page_size = parse_size_limit( + request.args.get("_size"), default=50, maximum=200 ) - if page < 1: - return Response.json({"error": "page must be >= 1"}, status=400) - if page_size < 1: - return Response.json({"error": "page_size must be >= 1"}, status=400) - max_page_size = 200 - if page_size > max_page_size: - page_size = max_page_size + except ValueError as ex: + return Response.error(str(ex), 400) offset = (page - 1) * page_size from datasette.utils.actions_sql import build_permission_rules_sql @@ -557,16 +569,17 @@ class PermissionRulesView(BaseView): def build_page_url(page_number): pairs = [] for key in request.args: - if key in {"page", "page_size"}: + if key in {"_page", "_size"}: continue for value in request.args.getlist(key): pairs.append((key, value)) - pairs.append(("page", str(page_number))) - pairs.append(("page_size", str(page_size))) + pairs.append(("_page", str(page_number))) + pairs.append(("_size", str(page_size))) query = urllib.parse.urlencode(pairs) return f"{request.path}?{query}" response = { + "ok": True, "action": action, "actor_id": (actor or {}).get("id") if actor else None, "page": page, @@ -587,17 +600,17 @@ class PermissionRulesView(BaseView): async def _check_permission_for_actor(ds, action, parent, child, actor): - """Shared logic for checking permissions. Returns a dict with check results.""" + """Shared logic for checking and explaining a permission decision.""" if action not in ds.actions: - return {"error": f"Unknown action: {action}"}, 404 + return error_body(f"Unknown action: {action}", 404), 404 if child and not parent: - return {"error": "parent is required when child is provided"}, 400 + return error_body("parent is required when child is provided", 400), 400 # Use the action's properties to create the appropriate resource object action_obj = ds.actions.get(action) if not action_obj: - return {"error": f"Unknown action: {action}"}, 400 + return error_body(f"Unknown action: {action}", 400), 400 # Global actions (no resource_class) don't have a resource if action_obj.resource_class is None: @@ -612,18 +625,32 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ - return {"error": f"Invalid action configuration: {action}"}, 500 + return error_body(f"Invalid action configuration: {action}", 500), 500 allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor) + from datasette.utils.actions_sql import explain_permission_for_resource + + explanation = await explain_permission_for_resource( + datasette=ds, + actor=actor, + action=action, + parent=parent, + child=child, + ) + response = { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, "action": action, "allowed": bool(allowed), + "actor": actor, "resource": { "parent": parent, "child": child, "path": _resource_path(parent, child), }, + "explanation": explanation, } if actor and "id" in actor: @@ -641,11 +668,25 @@ class PermissionCheckView(BaseView): as_format = request.url_vars.get("format") if not as_format: + actions = [ + { + "name": action.name, + "description": action.description, + "takes_parent": action.takes_parent, + "takes_child": action.takes_child, + "also_requires": action.also_requires, + } + for action in sorted( + self.ds.actions.values(), key=lambda action: action.name + ) + ] return await self.render( ["debug_check.html"], request, { - "sorted_actions": sorted(self.ds.actions.keys()), + "actions": actions, + "actor_json": request.args.get("actor") + or json.dumps(request.actor, indent=2), "has_debug_permission": True, }, ) @@ -653,13 +694,22 @@ class PermissionCheckView(BaseView): # JSON API - action parameter is required action = request.args.get("action") if not action: - return Response.json({"error": "action parameter is required"}, status=400) + return Response.error("action parameter is required", 400) parent = request.args.get("parent") child = request.args.get("child") + actor = request.actor + actor_json = request.args.get("actor") + if actor_json is not None: + try: + actor = json.loads(actor_json) + except json.JSONDecodeError as ex: + return Response.error(f"Invalid actor JSON: {ex}", 400) + if actor is not None and not isinstance(actor, dict): + return Response.error("actor must be a JSON object or null", 400) response, status = await _check_permission_for_actor( - self.ds, action, parent, child, request.actor + self.ds, action, parent, child, actor ) return Response.json(response, status=status) @@ -1200,7 +1250,7 @@ class JumpView(BaseView): match["display_name"] = row["display_name"] matches.append(match) - return Response.json({"matches": matches, "truncated": truncated}) + return Response.json({"ok": True, "matches": matches, "truncated": truncated}) class SchemaBaseView(BaseView): @@ -1222,7 +1272,7 @@ class SchemaBaseView(BaseView): headers = {} if self.ds.cors: add_cors_headers(headers) - return Response.json(data, headers=headers) + return Response.json({"ok": True, **data}, headers=headers) def format_error_response(self, error_message, format_, status=404): """Format error response based on requested format.""" @@ -1231,7 +1281,7 @@ class SchemaBaseView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response.json( - {"ok": False, "error": error_message}, status=status, headers=headers + error_body(error_message, status), status=status, headers=headers ) else: return Response.text(error_message, status=status) @@ -1307,17 +1357,17 @@ class DatabaseSchemaView(SchemaBaseView): database_name = request.url_vars["database"] format_ = request.url_vars.get("format") or "html" - # Check if database exists - if database_name not in self.ds.databases: - return self.format_error_response("Database not found", format_) - - # Check view-database permission + # Permission check comes first, so actors without view-database + # cannot distinguish existing databases from missing ones await self.ds.ensure_permission( action="view-database", resource=DatabaseResource(database=database_name), actor=request.actor, ) + if database_name not in self.ds.databases: + return self.format_error_response("Database not found", format_) + schema = await self.get_database_schema(database_name) if format_ == "json": @@ -1351,6 +1401,9 @@ class TableSchemaView(SchemaBaseView): actor=request.actor, ) + if database_name not in self.ds.databases: + return self.format_error_response("Database not found", format_) + # Get schema for the table db = self.ds.databases[database_name] result = await db.execute( diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 2753f876..d64f37d2 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -2,10 +2,10 @@ from urllib.parse import parse_qsl, urlencode from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict -from datasette.utils import sqlite3, tilde_decode +from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3, tilde_decode from datasette.utils.asgi import Response -from .base import BaseView, _error +from .base import BaseView from .query_helpers import ( QueryValidationError, _as_bool, @@ -34,12 +34,14 @@ class QueryParametersView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + return _block_framing( + Response.error(["Permission denied: need execute-sql"], 403) + ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - _error( + Response.error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) @@ -47,8 +49,16 @@ class QueryParametersView(BaseView): try: parameters = _derived_query_parameters(request.args.get("sql") or "") except QueryValidationError as ex: - return _block_framing(_error([ex.message], ex.status)) - return _block_framing(Response.json({"ok": True, "parameters": parameters})) + return _block_framing(Response.error([ex.message], ex.status)) + return _block_framing( + Response.json( + { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "parameters": parameters, + } + ) + ) def _query_list_url(path, query_string, *, set_args=None, remove_args=None): @@ -82,11 +92,12 @@ class QueryListView(BaseView): limit = _query_list_limit( request.args.get("_size"), default=20 if format_ == "html" else 50, + maximum=self.ds.max_returned_rows, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") is_private = _as_optional_bool(request.args.get("is_private"), "is_private") except QueryValidationError as ex: - return _error([ex.message], ex.status) + return Response.error([ex.message], ex.status) page = await self.ds.list_queries( database, @@ -111,9 +122,9 @@ class QueryListView(BaseView): if key != "_next" ] pairs.append(("_next", page.next)) - next_url = "{}?{}".format( - query_list_path, - urlencode(pairs), + next_url = self.ds.absolute_url( + request, + "{}?{}".format(request.path, urlencode(pairs)), ) current_filters = { @@ -199,7 +210,6 @@ class QueryListView(BaseView): "queries": page.queries, "next": page.next, "next_url": next_url, - "has_more": page.has_more, "limit": page.limit, "show_private_note": any(query.is_private for query in page.queries), "show_trusted_note": any(query.is_trusted for query in page.queries), @@ -298,28 +308,30 @@ class QueryCreateAnalyzeView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + return _block_framing( + Response.error(["Permission denied: need execute-sql"], 403) + ) if not await self.ds.allowed( action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need store-query"], 403)) + return _block_framing( + Response.error(["Permission denied: need store-query"], 403) + ) invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( - _error( + Response.error( ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], 400, ) ) sql = request.args.get("sql") or "" - return _block_framing( - Response.json( - await _query_create_analysis_data(self.ds, db, sql, request.actor) - ) - ) + analysis = await _query_create_analysis_data(self.ds, db, sql, request.actor) + analysis["unstable"] = UNSTABLE_API_MESSAGE + return _block_framing(Response.json(analysis)) class QueryStoreView(QueryCreateView): @@ -346,13 +358,13 @@ class QueryStoreView(QueryCreateView): resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need execute-sql"], 403) + return Response.error(["Permission denied: need execute-sql"], 403) if not await self.ds.allowed( action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need store-query"], 403) + return Response.error(["Permission denied: need store-query"], 403) is_json = False query_data = {} @@ -369,7 +381,7 @@ class QueryStoreView(QueryCreateView): return await self._error_response( request, db, query_data, ex.message, ex.status ) - return _error([ex.message], ex.status) + return Response.error([ex.message], ex.status) prepared.pop("analysis") name = prepared.pop("name") @@ -378,13 +390,18 @@ class QueryStoreView(QueryCreateView): except sqlite3.IntegrityError as ex: if not is_json and isinstance(query_data, dict): return await self._error_response(request, db, query_data, str(ex), 400) - return _error([str(ex)], 400) + return Response.error([str(ex)], 400) query = await self.ds.get_query(db.name, name) assert query is not None if is_json: return Response.json( - {"ok": True, "query": stored_query_to_dict(query)}, status=201 + { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "query": stored_query_to_dict(query), + }, + status=201, ) self.ds.add_message(request, "Query saved", self.ds.INFO) return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) @@ -398,14 +415,20 @@ class QueryDefinitionView(BaseView): query_name = tilde_decode(request.url_vars["query"]) query = await self.ds.get_query(db.name, query_name) if query is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="view-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied"], 403) - return Response.json({"ok": True, "query": stored_query_to_dict(query)}) + return Response.error(["Permission denied"], 403) + return Response.json( + { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "query": stored_query_to_dict(query), + } + ) class QueryUpdateView(BaseView): @@ -416,15 +439,17 @@ class QueryUpdateView(BaseView): query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied: need update-query"], 403) + return Response.error(["Permission denied: need update-query"], 403) if existing.is_trusted: - return _error(["Trusted queries cannot be updated using the API"], 403) + return Response.error( + ["Trusted queries cannot be updated using the API"], 403 + ) try: data, _ = await _json_or_form_payload(request) @@ -450,7 +475,7 @@ class QueryUpdateView(BaseView): self.ds, request, db, existing, update ) except QueryValidationError as ex: - return _error([ex.message], ex.status) + return Response.error([ex.message], ex.status) await self.ds.update_query(db.name, query_name, **update_kwargs) if data.get("return"): @@ -507,32 +532,32 @@ class QueryEditView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ) if existing.is_trusted: - return _error(["Trusted queries cannot be edited"], 403) + return Response.error(["Trusted queries cannot be edited"], 403) return await self._render_form(request, db, existing) async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied: need update-query"], 403) + return Response.error(["Permission denied: need update-query"], 403) if existing.is_trusted: - return _error(["Trusted queries cannot be edited"], 403) + return Response.error(["Trusted queries cannot be edited"], 403) data, _ = await _json_or_form_payload(request) if not isinstance(data, dict): - return _error(["Invalid form submission"], 400) + return Response.error(["Invalid form submission"], 400) sql = data.get("sql") sql = existing.sql if sql is None else sql.strip() title = data.get("title") or "" @@ -604,12 +629,16 @@ class QueryDeleteView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="delete-query", resource=QueryResource(db.name, query_name), actor=request.actor, ) + if existing.is_trusted: + return Response.error( + ["Trusted queries cannot be deleted using the API"], 403 + ) return await self.render( ["query_delete.html"], request, @@ -624,13 +653,17 @@ class QueryDeleteView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return _error(["Query not found: {}".format(query_name)], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="delete-query", resource=QueryResource(db.name, query_name), actor=request.actor, ): - return _error(["Permission denied: need delete-query"], 403) + return Response.error(["Permission denied: need delete-query"], 403) + if existing.is_trusted: + return Response.error( + ["Trusted queries cannot be deleted using the API"], 403 + ) data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) diff --git a/datasette/views/table.py b/datasette/views/table.py index 94d9de7b..f7edd744 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -22,9 +22,11 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, CustomRow, append_querystring, compound_keys_after_sql, + decode_write_json_rows, format_bytes, make_slot_function, tilde_encode, @@ -41,16 +43,24 @@ from datasette.utils import ( urlsafe_components, value_as_boolean, InvalidSql, + WriteJsonValueError, sqlite3, ) -from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response +from datasette.utils.asgi import ( + BadRequest, + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) from datasette.filters import Filters import sqlite_utils from dataclasses import dataclass, field from datasette.extras import ExtraScope from . import Context, from_extra -from .base import BaseView, DatasetteError, _error, stream_csv +from .base import BaseView, DatasetteError, stream_csv from .database import QueryView from .table_create_alter import ( ALTER_TABLE_COLUMN_TYPES, @@ -62,6 +72,7 @@ from .table_create_alter import ( from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, + count_is_truncated, precompute_database_action_permissions, precompute_table_action_permissions, resolve_table_extras, @@ -96,7 +107,6 @@ class TableContext(Context): human_description_en: str = from_extra() is_view: bool = from_extra() metadata: dict = from_extra() - next_url: str = from_extra() primary_keys: list = from_extra() private: bool = from_extra() query: dict = from_extra() @@ -113,6 +123,11 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + next_url: str = field( + metadata={ + "help": "Full URL for the next page of results, or None if there are no more pages. See :ref:`json_api_pagination`." + } + ) count_truncated: bool = field( metadata={ "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." @@ -205,7 +220,7 @@ class TableContext(Context): ) table_insert_ui: dict = field( metadata={ - "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys." + "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys." } ) table_alter_ui: dict = field( @@ -259,23 +274,18 @@ class Row: return json.dumps(d, default=repr, indent=2) -def row_label_from_label_columns(row, label_columns): - if not label_columns: +def row_label_from_label_column(row, label_column): + if not label_column: return None - values = [] - for label_column in label_columns: - try: - value = row[label_column] - except (KeyError, IndexError): - continue - if isinstance(value, dict): - value = value.get("label") - if value is None or value == "": - continue - values.append(str(value)) - if not values: + try: + value = row[label_column] + except (KeyError, IndexError): return None - return " ".join(values) + if isinstance(value, dict): + value = value.get("label") + if value is None or value == "": + return None + return str(value) async def run_sequential(*args): @@ -453,7 +463,6 @@ async def _table_page_data( is_view, table_insert_ui, table_alter_ui, - label_columns_ui, ): data = { "database": database_name, @@ -464,8 +473,6 @@ async def _table_page_data( data["insertRow"] = table_insert_ui if table_alter_ui: data["alterTable"] = table_alter_ui - if label_columns_ui: - data["labelColumns"] = label_columns_ui if not is_view: foreign_keys = await _foreign_key_autocomplete_urls( datasette, request, db, database_name, table_name @@ -488,8 +495,15 @@ async def _table_insert_ui( ): return None + can_update = await datasette.allowed( + action="update-row", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + column_types_map = await datasette.get_column_types(database_name, table_name) columns = [] + bulk_columns = [] column_details = await db.table_column_details(table_name) for column in column_details: if column.hidden: @@ -500,32 +514,40 @@ async def _table_insert_ui( and len(pks) == 1 and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER ) + column_type = column_types_map.get(column.name) + column_data = { + "name": column.name, + "sqlite_type": _column_sqlite_type_for_insert_form(column), + "notnull": column.notnull, + "default": column.default_value, + "has_default": column.default_value is not None, + "is_pk": is_pk, + "is_auto_pk": is_auto_pk, + "value_kind": _column_value_kind_for_insert_form(column), + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + bulk_columns.append(column_data) if is_auto_pk: continue - column_type = column_types_map.get(column.name) - columns.append( - { - "name": column.name, - "sqlite_type": _column_sqlite_type_for_insert_form(column), - "notnull": column.notnull, - "default": column.default_value, - "has_default": column.default_value is not None, - "is_pk": is_pk, - "value_kind": _column_value_kind_for_insert_form(column), - "column_type": ( - {"type": column_type.name, "config": column_type.config} - if column_type is not None - else None - ), - } - ) + columns.append(column_data) - return { + data = { "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, + "bulkColumns": bulk_columns, "primaryKeys": pks, + "maxInsertRows": datasette.setting("max_insert_rows"), } + if can_update: + data["upsertPath"] = "{}/-/upsert".format( + datasette.urls.table(database_name, table_name) + ) + return data async def _table_alter_ui( @@ -614,29 +636,6 @@ async def _table_alter_ui( return data -async def _table_label_columns_ui(datasette, request, db, database_name, table_name): - if not await datasette.allowed( - action="set-label-columns", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return None - - column_details = await db.table_column_details(table_name) - columns = [column.name for column in column_details if not column.hidden] - current = await db.label_columns_for_table(table_name) - explicit = await datasette.get_label_columns(database_name, table_name) - return { - "path": "{}/-/set-label-columns".format( - datasette.urls.table(database_name, table_name) - ), - "tableName": table_name, - "columns": columns, - "current": current, - "isOverridden": explicit is not None, - } - - async def display_columns_and_rows( datasette, database_name, @@ -676,9 +675,9 @@ async def display_columns_and_rows( pks_for_display = pks if not pks_for_display: pks_for_display = ["rowid"] - label_columns = None + label_column = None if link_column: - label_columns = await db.label_columns_for_table(table_name) + label_column = await db.label_column_for_table(table_name) row_action_permissions = {} if link_column and request is not None and db.is_mutable: row_action_permissions = await datasette.allowed_many( @@ -726,7 +725,7 @@ async def display_columns_and_rows( is_special_link_column = len(pks) != 1 pk_path = path_from_row_pks(row, pks, not pks, False) row_path = path_from_row_pks(row, pks, not pks) - row_label = row_label_from_label_columns(row, label_columns) + row_label = row_label_from_label_column(row, label_column) row_action_label = pk_path if row_label and row_label != pk_path: row_action_label = "{} {}".format(pk_path, row_label) @@ -956,9 +955,7 @@ class TableInsertView(BaseView): def _errors(errors): return None, errors, {} - if not request.headers.get("content-type").startswith("application/json"): - # TODO: handle form-encoded data - return _errors(["Invalid content-type, must be application/json"]) + # The body is parsed as JSON regardless of the Content-Type header try: data = await request.json() except json.JSONDecodeError as e: @@ -1042,7 +1039,7 @@ class TableInsertView(BaseView): try: resolved = await self.ds.resolve_table(request) except NotFound as e: - return _error([e.args[0]], 404) + return Response.error([e.args[0]], 404) db = resolved.db database_name = db.name table_name = resolved.table @@ -1050,7 +1047,7 @@ class TableInsertView(BaseView): # Table must exist (may handle table creation in the future) db = self.ds.get_database(database_name) if not await db.table_exists(table_name): - return _error(["Table not found: {}".format(table_name)], 404) + return Response.error(["Table not found: {}".format(table_name)], 404) if upsert: # Must have insert-row AND upsert-row permissions @@ -1066,7 +1063,7 @@ class TableInsertView(BaseView): actor=request.actor, ) ): - return _error( + return Response.error( ["Permission denied: need both insert-row and update-row"], 403 ) else: @@ -1076,25 +1073,32 @@ class TableInsertView(BaseView): resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(["Permission denied"], 403) + return Response.error(["Permission denied"], 403) if not db.is_mutable: - return _error(["Database is immutable"], 403) + return Response.error(["Database is immutable"], 403) pks = await db.primary_keys(table_name) - rows, errors, extras = await self._validate_data( - request, db, table_name, pks, upsert - ) + try: + rows, errors, extras = await self._validate_data( + request, db, table_name, pks, upsert + ) + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if errors: - return _error(errors, 400) + return Response.error(errors, 400) + try: + rows = decode_write_json_rows(rows) + except WriteJsonValueError as e: + return Response.error([str(e)], 400) # Validate column types ct_errors = await _validate_column_types( self.ds, database_name, table_name, rows ) if ct_errors: - return _error(ct_errors, 400) + return Response.error(ct_errors, 400) num_rows = len(rows) @@ -1108,14 +1112,16 @@ class TableInsertView(BaseView): alter = extras.get("alter") if upsert and (ignore or replace): - return _error(["Upsert does not support ignore or replace"], 400) + return Response.error(["Upsert does not support ignore or replace"], 400) if replace and not await self.ds.allowed( action="update-row", resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(['Permission denied: need update-row to use "replace"'], 403) + return Response.error( + ['Permission denied: need update-row to use "replace"'], 403 + ) initial_schema = None if alter: @@ -1125,7 +1131,7 @@ class TableInsertView(BaseView): resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(["Permission denied for alter-table"], 403) + return Response.error(["Permission denied for alter-table"], 403) # Track initial schema to check if it changed later initial_schema = await db.execute_fn( lambda conn: sqlite_utils.Database(conn)[table_name].schema @@ -1165,7 +1171,7 @@ class TableInsertView(BaseView): try: rows = await db.execute_write_fn(insert_or_upsert_rows, request=request) except Exception as e: - return _error([str(e)]) + return Response.error([str(e)]) result = {"ok": True} if should_return: if upsert: @@ -1222,7 +1228,11 @@ class TableInsertView(BaseView): ) ) - return Response.json(result, status=200 if upsert else 201) + return Response.json( + result, + status=200 if upsert else 201, + default=CustomJSONEncoder().default, + ) class TableUpsertView(TableInsertView): @@ -1242,7 +1252,7 @@ class TableSetColumnTypeView(BaseView): try: resolved = await self.ds.resolve_table(request) except NotFound as e: - return _error([e.args[0]], 404) + return Response.error([e.args[0]], 404) database_name = resolved.db.name table_name = resolved.table @@ -1252,41 +1262,39 @@ class TableSetColumnTypeView(BaseView): resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(["Permission denied"], 403) - - content_type = request.headers.get("content-type") or "" - if not content_type.startswith("application/json"): - return _error(["Invalid content-type, must be application/json"], 400) + return Response.error(["Permission denied"], 403) try: data = await request.json() except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)], 400) + return Response.error(["Invalid JSON: {}".format(e)], 400) + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if not isinstance(data, dict): - return _error(["JSON must be a dictionary"], 400) + return Response.error(["JSON must be a dictionary"], 400) invalid_keys = set(data.keys()) - {"column", "column_type"} if invalid_keys: - return _error( + return Response.error( ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))], 400, ) if "column" not in data: - return _error(['"column" is required'], 400) + return Response.error(['"column" is required'], 400) column = data["column"] if not isinstance(column, str): - return _error(['"column" must be a string'], 400) + return Response.error(['"column" must be a string'], 400) if "column_type" not in data: - return _error(['"column_type" is required'], 400) + return Response.error(['"column_type" is required'], 400) column_details = await self.ds._get_resource_column_details( database_name, table_name ) if column not in column_details: - return _error(["Column not found: {}".format(column)], 400) + return Response.error(["Column not found: {}".format(column)], 400) column_type_data = data["column_type"] if column_type_data is None: @@ -1303,11 +1311,11 @@ class TableSetColumnTypeView(BaseView): ) if not isinstance(column_type_data, dict): - return _error(['"column_type" must be an object or null'], 400) + return Response.error(['"column_type" must be an object or null'], 400) invalid_column_type_keys = set(column_type_data.keys()) - {"type", "config"} if invalid_column_type_keys: - return _error( + return Response.error( [ 'Invalid column_type parameter: "{}"'.format( '", "'.join(sorted(invalid_column_type_keys)) @@ -1317,24 +1325,24 @@ class TableSetColumnTypeView(BaseView): ) if "type" not in column_type_data: - return _error(['"column_type.type" is required'], 400) + return Response.error(['"column_type.type" is required'], 400) column_type = column_type_data["type"] if not isinstance(column_type, str): - return _error(['"column_type.type" must be a string'], 400) + return Response.error(['"column_type.type" must be a string'], 400) config = column_type_data.get("config") if config is not None and not isinstance(config, dict): - return _error(['"column_type.config" must be a dictionary'], 400) + return Response.error(['"column_type.config" must be a dictionary'], 400) if column_type not in self.ds._column_types: - return _error(["Unknown column type: {}".format(column_type)], 400) + return Response.error(["Unknown column type: {}".format(column_type)], 400) try: await self.ds.set_column_type( database_name, table_name, column, column_type, config ) except ValueError as e: - return _error([str(e)], 400) + return Response.error([str(e)], 400) return Response.json( { @@ -1348,95 +1356,6 @@ class TableSetColumnTypeView(BaseView): ) -class TableSetLabelColumnsView(BaseView): - name = "table-set-label-columns" - - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - try: - resolved = await self.ds.resolve_table(request) - except NotFound as e: - return _error([e.args[0]], 404) - - database_name = resolved.db.name - table_name = resolved.table - - if not await self.ds.allowed( - action="set-label-columns", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return _error(["Permission denied"], 403) - - content_type = request.headers.get("content-type") or "" - if not content_type.startswith("application/json"): - return _error(["Invalid content-type, must be application/json"], 400) - - try: - data = await request.json() - except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)], 400) - - if not isinstance(data, dict): - return _error(["JSON must be a dictionary"], 400) - - invalid_keys = set(data.keys()) - {"columns"} - if invalid_keys: - return _error( - ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))], - 400, - ) - - if "columns" not in data: - return _error(['"columns" is required'], 400) - columns = data["columns"] - - if columns is None: - await self.ds.remove_label_columns(database_name, table_name) - return Response.json( - { - "ok": True, - "database": database_name, - "table": table_name, - "columns": None, - }, - status=200, - ) - - if ( - not isinstance(columns, list) - or not columns - or not all(isinstance(c, str) for c in columns) - ): - return _error( - ['"columns" must be a non-empty list of strings, or null'], 400 - ) - - if len(set(columns)) != len(columns): - return _error(['"columns" must not contain duplicates'], 400) - - column_details = await self.ds._get_resource_column_details( - database_name, table_name - ) - for column in columns: - if column not in column_details: - return _error(["Column not found: {}".format(column)], 400) - - await self.ds.set_label_columns(database_name, table_name, columns) - - return Response.json( - { - "ok": True, - "database": database_name, - "table": table_name, - "columns": columns, - }, - status=200, - ) - - class TableDropView(BaseView): name = "table-drop" @@ -1447,28 +1366,30 @@ class TableDropView(BaseView): try: resolved = await self.ds.resolve_table(request) except NotFound as e: - return _error([e.args[0]], 404) + return Response.error([e.args[0]], 404) db = resolved.db database_name = db.name table_name = resolved.table # Table must exist db = self.ds.get_database(database_name) if not await db.table_exists(table_name): - return _error(["Table not found: {}".format(table_name)], 404) + return Response.error(["Table not found: {}".format(table_name)], 404) if not await self.ds.allowed( action="drop-table", resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(["Permission denied"], 403) + return Response.error(["Permission denied"], 403) if not db.is_mutable: - return _error(["Database is immutable"], 403) + return Response.error(["Database is immutable"], 403) confirm = False try: data = await request.json() confirm = data.get("confirm") except json.JSONDecodeError: pass + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if not confirm: return Response.json( @@ -1563,7 +1484,7 @@ def _autocomplete_prefix_like(column): return "{} like :prefix escape char(92)".format(escape_sqlite(column)) -def _autocomplete_order_by(pks, label_columns, exact_pk, label_matches_first=True): +def _autocomplete_order_by(pks, label_column, exact_pk, label_matches_first=True): clauses = [] if exact_pk: clauses.append( @@ -1571,18 +1492,15 @@ def _autocomplete_order_by(pks, label_columns, exact_pk, label_matches_first=Tru escape_sqlite(pks[0]) ) ) - if label_columns: - label_likes = [_autocomplete_like(column) for column in label_columns] - any_label_like = " or ".join(label_likes) + if label_column: + label_like = _autocomplete_like(label_column) if label_matches_first: - clauses.append("case when {} then 0 else 1 end".format(any_label_like)) - total_length = " + ".join( - "case when {} then length(cast({} as text)) else 0 end".format( - label_like, escape_sqlite(column) + clauses.append("case when {} then 0 else 1 end".format(label_like)) + clauses.append( + "case when {} then length(cast({} as text)) end".format( + label_like, escape_sqlite(label_column) ) - for label_like, column in zip(label_likes, label_columns) ) - clauses.append("case when {} then {} end".format(any_label_like, total_length)) else: clauses.append("length(cast({} as text))".format(escape_sqlite(pks[0]))) clauses.extend(escape_sqlite(pk) for pk in pks) @@ -1599,15 +1517,12 @@ def _autocomplete_initial_order_by(pks): return ", ".join(order_by) -def _autocomplete_response_rows(rows, pks, label_columns): +def _autocomplete_response_rows(rows, pks, label_column): response_rows = [] for row in rows: item = {"pks": {pk: row[pk] for pk in pks}} - if label_columns: - if len(label_columns) == 1: - item["label"] = row[label_columns[0]] - else: - item["label"] = row_label_from_label_columns(row, label_columns) + if label_column: + item["label"] = row[label_column] response_rows.append(item) return response_rows @@ -1637,8 +1552,10 @@ class TableAutocompleteView(BaseView): pks = await db.primary_keys(table_name) if not pks: pks = ["rowid"] - label_columns = await db.label_columns_for_table(table_name) - select_columns = list(dict.fromkeys(pks + label_columns)) + label_column = await db.label_column_for_table(table_name) + select_columns = list( + dict.fromkeys(pks + ([label_column] if label_column else [])) + ) select_sql = ", ".join(escape_sqlite(column) for column in select_columns) q = request.args.get("q") or "" initial_arg = request.args.get("_initial") @@ -1649,7 +1566,7 @@ class TableAutocompleteView(BaseView): and value_as_boolean(initial_arg) ) if not q and not initial: - return Response.json({"rows": []}) + return Response.json({"ok": True, "rows": []}) params = { "q": q, "like": "%{}%".format(_escape_like(q)), @@ -1657,12 +1574,11 @@ class TableAutocompleteView(BaseView): } like_columns = pks[:] - for label_column in label_columns: - if label_column not in like_columns: - like_columns.append(label_column) + if label_column and label_column not in like_columns: + like_columns.append(label_column) where_sql = " or ".join(_autocomplete_like(column) for column in like_columns) exact_pk = len(pks) == 1 - order_by = _autocomplete_order_by(pks, label_columns, exact_pk) + order_by = _autocomplete_order_by(pks, label_column, exact_pk) if initial: where_sql = "1 = 1" @@ -1713,10 +1629,13 @@ class TableAutocompleteView(BaseView): custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS, ) except QueryInterrupted: - return Response.json({"rows": []}) + return Response.json({"ok": True, "rows": []}) return Response.json( - {"rows": _autocomplete_response_rows(results.rows, pks, label_columns)} + { + "ok": True, + "rows": _autocomplete_response_rows(results.rows, pks, label_column), + } ) @@ -2301,11 +2220,11 @@ async def table_view_data( # Expand labeled columns if requested expanded_columns = [] - # List of (fk_dict, label_columns-or-None) pairs for that table + # List of (fk_dict, label_column-or-None) pairs for that table expandable_columns = [] for fk in await db.foreign_keys_for_table(table_name): - label_columns = await db.label_columns_for_table(fk["other_table"]) - expandable_columns.append((fk, label_columns or None)) + label_column = await db.label_column_for_table(fk["other_table"]) + expandable_columns.append((fk, label_column)) columns_to_expand = None try: @@ -2375,10 +2294,16 @@ async def table_view_data( # Resolve extras extras = extra_names_from_request(request) + if not extra_extras: + # Data formats reject unknown extras; the HTML path (which passes + # extra_extras={"_html"}) resolves internal extras of its own + table_extra_registry.validate_requested(extras, ExtraScope.TABLE) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): extras.add("facet_results") if request.args.get("_shape") == "object": extras.add("primary_keys") + if "count" in extras: + extras.add("count_truncated") if extra_extras: extras.update(extra_extras) @@ -2433,6 +2358,7 @@ async def table_view_data( data = { "ok": True, "next": next_value and str(next_value) or None, + "next_url": next_url, } data.update( await resolve_table_extras( @@ -2457,7 +2383,7 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: - data["count_truncated"] = _count_truncated_for_table_page( + data["count_truncated"] = count_is_truncated( datasette, db, database_name, table_name, count_sql, data.get("count") ) data.update(extra_context_from_filters) @@ -2509,9 +2435,6 @@ async def table_view_data( table_alter_ui = await _table_alter_ui( datasette, request, db, database_name, table_name, is_view, pks ) - label_columns_ui = await _table_label_columns_ui( - datasette, request, db, database_name, table_name - ) data["table_insert_ui"] = table_insert_ui data["table_alter_ui"] = table_alter_ui data["table_page_data"] = await _table_page_data( @@ -2523,30 +2446,11 @@ async def table_view_data( is_view=is_view, table_insert_ui=table_insert_ui, table_alter_ui=table_alter_ui, - label_columns_ui=label_columns_ui, ) return data, rows[:page_size], columns, expanded_columns, sql, next_url -def _count_truncated_for_table_page( - datasette, db, database_name, table_name, count_sql, count -): - if count != db.count_limit + 1: - return False - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - try: - datasette.inspect_data[database_name]["tables"][table_name]["count"] - return False - except KeyError: - pass - return True - - async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index ffeb7f14..4deeafcc 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -20,14 +20,16 @@ from datasette.column_types import SQLiteType from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( + decode_write_json_rows, escape_sqlite, get_outbound_foreign_keys, table_column_details, + WriteJsonValueError, ) -from datasette.utils.asgi import NotFound, Response +from datasette.utils.asgi import NotFound, PayloadTooLarge, Response from datasette.utils.sqlite import sqlite_hidden_table_names -from .base import BaseView, _error +from .base import BaseView CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] CREATE_TABLE_SQLITE_TYPES = { @@ -267,6 +269,11 @@ async def _create_table_ui_context( "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, "defaultExpressions": default_expression_options(), + "canInsertRows": await datasette.allowed( + action="insert-row", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ), } can_set_column_type = await datasette.allowed( action="set-column-type", @@ -798,20 +805,22 @@ class TableCreateView(BaseView): resource=DatabaseResource(database=database_name), actor=request.actor, ): - return _error(["Permission denied"], 403) + return Response.error(["Permission denied"], 403) try: data = await request.json() except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)]) + return Response.error(["Invalid JSON: {}".format(e)]) + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if not isinstance(data, dict): - return _error(["JSON must be an object"]) + return Response.error(["JSON must be an object"]) try: create_request = CreateTableRequest.model_validate(data) except ValidationError as e: - return _error(_create_table_pydantic_errors(e)) + return Response.error(_create_table_pydantic_errors(e)) ignore = create_request.ignore replace = create_request.replace @@ -823,7 +832,7 @@ class TableCreateView(BaseView): resource=DatabaseResource(database=database_name), actor=request.actor, ): - return _error(["Permission denied: need update-row"], 403) + return Response.error(["Permission denied: need update-row"], 403) table_name = create_request.table table_exists = await db.table_exists(table_name) @@ -837,7 +846,11 @@ class TableCreateView(BaseView): resource=DatabaseResource(database=database_name), actor=request.actor, ): - return _error(["Permission denied: need insert-row"], 403) + return Response.error(["Permission denied: need insert-row"], 403) + try: + rows = decode_write_json_rows(rows) + except WriteJsonValueError as e: + return Response.error([str(e)], 400) alter = False if rows: @@ -852,7 +865,9 @@ class TableCreateView(BaseView): resource=DatabaseResource(database=database_name), actor=request.actor, ): - return _error(["Permission denied: need alter-table"], 403) + return Response.error( + ["Permission denied: need alter-table"], 403 + ) alter = True pk = create_request.pk @@ -868,7 +883,7 @@ class TableCreateView(BaseView): elif len(actual_pks) > 1 and pks and set(pks) != set(actual_pks): bad_pks = True if bad_pks: - return _error(["pk cannot be changed for existing table"]) + return Response.error(["pk cannot be changed for existing table"]) pks = actual_pks initial_schema = None @@ -911,7 +926,7 @@ class TableCreateView(BaseView): try: schema = await db.execute_write_fn(create_table, request=request) except Exception as e: - return _error([str(e)]) + return Response.error([str(e)]) if initial_schema is not None and initial_schema != schema: await self.ds.track_event( @@ -986,7 +1001,7 @@ class DatabaseForeignKeyTargetsView(BaseView): actor=request.actor, ) if not (can_create_table or can_alter_table): - return _error(["Permission denied: need create-table"], 403) + return Response.error(["Permission denied: need create-table"], 403) hidden_tables = await db.execute_fn( lambda conn: set(sqlite_hidden_table_names(conn)) @@ -1015,21 +1030,21 @@ class TableForeignKeySuggestionsView(BaseView): try: resolved = await self.ds.resolve_table(request) except NotFound as e: - return _error([e.args[0]], 404) + return Response.error([e.args[0]], 404) db = resolved.db database_name = db.name table_name = resolved.table if resolved.is_view: - return _error(["Cannot suggest foreign keys for a view"], 400) + return Response.error(["Cannot suggest foreign keys for a view"], 400) if not await self.ds.allowed( action="alter-table", resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(["Permission denied: need alter-table"], 403) + return Response.error(["Permission denied: need alter-table"], 403) source_columns, targets, current_by_column = await db.execute_fn( lambda conn: _foreign_key_suggestion_metadata(conn, table_name) @@ -1137,7 +1152,7 @@ class TableAlterView(BaseView): try: resolved = await self.ds.resolve_table(request) except NotFound as e: - return _error([e.args[0]], 404) + return Response.error([e.args[0]], 404) db = resolved.db database_name = db.name @@ -1148,27 +1163,25 @@ class TableAlterView(BaseView): resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): - return _error(["Permission denied: need alter-table"], 403) + return Response.error(["Permission denied: need alter-table"], 403) if not db.is_mutable: - return _error(["Database is immutable"], 403) - - content_type = request.headers.get("content-type") or "" - if not content_type.startswith("application/json"): - return _error(["Invalid content-type, must be application/json"], 400) + return Response.error(["Database is immutable"], 403) try: data = await request.json() except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)], 400) + return Response.error(["Invalid JSON: {}".format(e)], 400) + except PayloadTooLarge as e: + return Response.error([str(e)], 413) if not isinstance(data, dict): - return _error(["JSON must be a dictionary"], 400) + return Response.error(["JSON must be a dictionary"], 400) try: alter_request = AlterTableRequest.model_validate(data) except ValidationError as e: - return _error(_pydantic_errors(e), 400) + return Response.error(_pydantic_errors(e), 400) def alter_table(conn): before_schema = _table_schema_from_conn(conn, table_name) @@ -1317,7 +1330,7 @@ class TableAlterView(BaseView): alter_table, request=request ) except Exception as e: - return _error([str(e)], 400) + return Response.error([str(e)], 400) altered = before_schema != after_schema if altered: diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 221cb02b..50002d92 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -1,6 +1,7 @@ import itertools from dataclasses import dataclass +from datasette.column_types import SQLiteType from datasette.database import QueryInterrupted from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm @@ -140,6 +141,39 @@ class CountExtra(Extra): return count +def count_is_truncated(datasette, db, database_name, table_name, count_sql, count): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + +class CountTruncatedExtra(Extra): + description = "True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count." + example = ExtraExample("/fixtures/facetable.json?_extra=count,count_truncated") + scopes = {ExtraScope.TABLE} + expensive = True + + async def resolve(self, context, count): + return count_is_truncated( + context.datasette, + context.db, + context.database_name, + context.table_name, + context.count_sql, + count, + ) + + class FacetInstancesProvider(Provider): scopes = {ExtraScope.TABLE} @@ -287,21 +321,6 @@ class HumanDescriptionEnExtra(Extra): return human_description_en -class NextUrlExtra(Extra): - description = "Full URL for the next page of results" - example = ExtraExample( - "/fixtures/facetable.json?_size=1&_extra=next_url", - note=( - "``null`` if there are no more pages of results. " - "See :ref:`json_api_pagination`." - ), - ) - scopes = {ExtraScope.TABLE} - - async def resolve(self, context): - return context.next_url - - class ColumnsExtra(Extra): description = "List of column names returned by this table, row or query." example = ExtraExample("/fixtures/facetable.json?_extra=columns") @@ -342,6 +361,55 @@ class PrimaryKeysExtra(Extra): return context.pks +def column_detail_as_json(column): + return { + "type": column.type, + "sqlite_type": SQLiteType.from_declared_type(column.type).value, + "notnull": bool(column.notnull), + "default": column.default_value, + "is_pk": bool(column.is_pk), + "pk_position": column.is_pk, + "hidden": column.hidden, + } + + +class ColumnDetailsExtra(Extra): + description = ( + "SQLite schema details for columns in this table. The dictionary maps " + "column names to objects describing the schema for each column." + ) + docs_note = ( + "Each object has ``type`` as the declared type string returned by " + 'SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the ' + "normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, " + "``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` " + 'as the raw SQL default expression string, such as ``"42"``, ' + "``\"'hello'\"`` or ``\"datetime('now')\"``, or ``null`` if there is " + "no default; ``is_pk`` as a boolean; ``pk_position`` as the integer " + "primary key position reported by SQLite, or ``0`` for columns that " + "are not part of the primary key; and ``hidden`` as the integer value " + "reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for " + "normal columns, ``1`` for hidden virtual table columns, ``2`` for " + "virtual generated columns and ``3`` for stored generated columns." + ) + example = ExtraExample("/fixtures/binary_data.json?_size=0&_extra=column_details") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/binary_data/1.json?_extra=column_details" + ) + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW} + + async def resolve(self, context): + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + return { + column_name: column_detail_as_json(column) + for column_name, column in column_details.items() + } + + class ActionsExtra(Extra): description = 'Async callable returning table or view actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions`, :ref:`plugin_hook_table_actions` and :ref:`plugin_hook_view_actions`.' scopes = {ExtraScope.TABLE} @@ -1070,15 +1138,14 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): - description = "List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_columns)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_columns`` is a list of label column(s) in the referenced table, or ``None``." + description = "List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_column)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_column`` is the label column in the referenced table, or ``None``." docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels." example = ExtraExample( "/fixtures/facetable.json?_extra=expandable_columns", note=( - "Each item is a ``[foreign_key, label_columns]`` pair: the " - "foreign key relationship, then the list of column(s) in the " - "other table that would be used as the label for each expanded " - "value." + "Each item is a ``[foreign_key, label_column]`` pair: the " + "foreign key relationship, then the column in the other table " + "that would be used as the label for each expanded value." ), ) scopes = {ExtraScope.TABLE} @@ -1087,8 +1154,8 @@ class ExpandableColumnsExtra(Extra): expandables = [] db = context.datasette.databases[context.database_name] for fk in await db.foreign_keys_for_table(context.table_name): - label_columns = await db.label_columns_for_table(fk["other_table"]) - expandables.append((fk, label_columns or None)) + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) return expandables @@ -1168,7 +1235,6 @@ TABLE_EXTRA_BUNDLES = { "count", "count_sql", "human_description_en", - "next_url", "metadata", "query", "columns", @@ -1197,16 +1263,17 @@ TABLE_EXTRA_BUNDLES = { TABLE_EXTRA_CLASSES = [ CountExtra, + CountTruncatedExtra, CountSqlExtra, FacetResultsExtra, FacetsTimedOutExtra, SuggestedFacetsExtra, FacetInstancesProvider, HumanDescriptionEnExtra, - NextUrlExtra, ColumnsExtra, AllColumnsExtra, PrimaryKeysExtra, + ColumnDetailsExtra, DisplayColumnsAndRowsProvider, DisplayColumnsExtra, DisplayRowsExtra, diff --git a/datasette/write_sql.py b/datasette/write_sql.py index cdc0c6d3..ca144a72 100644 --- a/datasette/write_sql.py +++ b/datasette/write_sql.py @@ -138,6 +138,33 @@ def decision_for_write_sql_operation( ), ) ) + if operation.operation == "create" and operation.target_type == "view": + if operation.database is None: + return UnsupportedWriteSqlOperation(unsupported_message) + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="create-view", + resource=DatabaseResource(database=operation.database), + ), + ) + ) + if ( + operation.operation == "drop" + and operation.target_type == "view" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="drop-view", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) if ( operation.operation == "alter" and operation.target_type == "table" diff --git a/docs/authentication.rst b/docs/authentication.rst index 9d97a3b9..d720c4db 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -45,12 +45,12 @@ Using the "root" actor Datasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github `__ for example. -The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user has **all permissions** - they can perform any action regardless of other permission rules. +The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user starts with **all permissions**: Datasette contributes a global allow rule for every action. More specific deny rules can still override that global rule. The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``set-label-columns``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``create-view``, ``alter-table``, ``set-column-type``, ``drop-table``, ``drop-view``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -84,12 +84,12 @@ Click on that link and then visit ``http://127.0.0.1:8001/-/actor`` to confirm t Permissions =========== -Datasette's permissions system is built around SQL queries. Datasette and its plugins construct SQL queries to resolve the list of resources that an actor cas access. - The key question the permissions system answers is this: Is this **actor** allowed to perform this **action**, optionally against this particular **resource**? +Every permission decision can be understood in terms of those three values. Datasette implements the decisions using SQL, but you do not need to understand the generated SQL to configure or debug permissions. + **Actors** are :ref:`described above `. An **action** is a string describing the action the actor would like to perform. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``. @@ -138,7 +138,51 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Permission rules describe an effect (``allow`` or ``deny``) at one of three levels: + +``resource`` + A specific child resource, such as the ``analytics/sales`` table. + +``parent`` + A parent resource, such as the ``analytics`` database. A parent rule also applies to its child resources. + +``global`` + Every resource for that action. + +Datasette resolves matching rules from most specific to least specific: + +#. Resource rules take precedence over parent and global rules. +#. Parent rules take precedence over global rules. +#. If both allow and deny rules match at the same level, deny takes precedence. +#. If no rule matches, access is denied. + +This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny. + +.. list-table:: Permission rule examples + :header-rows: 1 + + * - Matching rules + - Result + - Explanation + * - Global allow + - Allow + - The global rule is the most specific matching rule. + * - Global allow, parent deny + - Deny + - The parent rule is more specific. + * - Parent deny, resource allow + - Allow + - The resource rule is more specific. + * - Resource allow and resource deny + - Deny + - Deny takes precedence at the same level. + * - No matching rules + - Deny + - Permissions default to deny when no rule applies. + +The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules. + +Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -149,12 +193,12 @@ resources were allowed or denied. The combined sources are: * ``allow`` blocks configured in :ref:`datasette.yaml `. * :ref:`Actor restrictions ` encoded into the actor dictionary or API token. -* The "root" user shortcut when ``--root`` (or :attr:`Datasette.root_enabled `) is active, replying ``True`` to all permission chucks unless configuration rules deny them at a more specific level. +* The "root" user rule when ``--root`` (or :attr:`Datasette.root_enabled `) is active. This is a global allow rule, so a more specific configuration deny can override it. * Any additional SQL provided by plugins implementing :ref:`plugin_hook_permission_resources_sql`. -Datasette evaluates the SQL to determine if the requested ``resource`` is -included. Explicit deny rules returned by configuration or plugins will block -access even if other rules allowed it. +Actor restrictions are applied after the allow/deny rules. They act as an additional allowlist: a restriction can remove access but cannot grant access that the actor did not already have. See :ref:`authentication_cli_create_token_restrict`. + +Some actions have dependencies on other actions. These are evaluated as an ``AND`` condition. For example, ``execute-sql`` also requires ``view-database``: both decisions must be allowed for the final result to be allowed. .. _authentication_permissions_allow: @@ -903,7 +947,7 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } .. [[[end]]] -Other table-scoped write permissions, including ``set-column-type`` and ``set-label-columns``, can be configured in the same place. +Other table-scoped write permissions, including ``set-column-type``, can be configured in the same place. And for ``insert-row`` against the ``reports`` table in that ``docs`` database: @@ -991,7 +1035,9 @@ The ``/-/create-token`` page cannot be accessed by actors that are authenticated Datasette plugins that implement their own form of API token authentication should follow this convention. -You can disable the signed token feature entirely using the :ref:`allow_signed_tokens ` setting. +If a request presents a token that a token handler recognizes but rejects - an invalid signature, a malformed payload or an expired token - Datasette responds with a ``401`` status, the :ref:`standard JSON error format ` and a ``WWW-Authenticate: Bearer error="invalid_token"`` header. This means API clients can distinguish "your token needs to be renewed" (``401``) from "your token does not grant this permission" (``403``). A ``Bearer`` token that no registered handler recognizes at all is ignored, since it may be intended for an authentication plugin. + +You can disable the signed token feature entirely using the :ref:`allow_signed_tokens ` setting. Requests presenting a ``dstok_`` token while the feature is disabled receive a ``401``. .. _authentication_cli_create_token: @@ -1143,11 +1189,23 @@ The debug tool at ``/-/permissions`` is available to any actor with the ``permis datasette -s permissions.permissions-debug true data.db -The page shows the permission checks that have been carried out by the Datasette instance. +The permission debug tools answer four different questions: -It also provides an interface for running hypothetical permission checks against a hypothetical actor. This is a useful way of confirming that your configured permissions work in the way you expect. +Why was this decision allowed or denied? + Use :ref:`PermissionCheckView`. It shows every matching rule, identifies the winning specificity level, applies actor restrictions and evaluates any required actions. -This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system. +Which resources can the current actor access? + Use :ref:`AllowedResourcesView` to view an access map for a selected action. + +Which raw rules did Datasette and its plugins contribute? + Use :ref:`PermissionRulesView` to inspect the rules before they are resolved into decisions. + +Which checks has this Datasette instance performed recently? + Use ``/-/permissions`` to view recent permission activity. + +These tools are designed to help administrators and plugin authors understand and confirm the effective permissions configuration. + +These debug endpoints are exempt from the :ref:`JSON API stability promise ` - their JSON shapes may change in future releases. .. _AllowedResourcesView: @@ -1158,7 +1216,7 @@ The ``/-/allowed`` endpoint displays resources that the current actor can access This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/allowed.json``) to get the raw JSON response instead. -Pass ``?action=view-table`` (or another action) to select the action. Optional ``parent=`` and ``child=`` query parameters can narrow the results to a specific database/table pair. +Pass ``?action=view-table`` (or another action) to select the action. Optional ``parent=`` and ``child=`` query parameters can narrow the results to a specific database/table pair. Results are paginated: ``?_size=`` sets the page size (default 50, maximum 200, ``max`` for the maximum) and ``?_page=`` selects a page. This endpoint is publicly accessible to help users understand their own permissions. The potentially sensitive ``reason`` field is only shown to users with the ``permissions-debug`` permission - it shows the plugins and explanatory reasons that were responsible for each decision. @@ -1171,7 +1229,7 @@ The ``/-/rules`` endpoint displays all permission rules (both allow and deny) fo This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/rules.json?action=view-table``) to get the raw JSON response instead. -Pass ``?action=`` as a query parameter to specify which action to check. +Pass ``?action=`` as a query parameter to specify which action to check. The ``?_size=`` and ``?_page=`` pagination parameters work the same as on ``/-/allowed``. This endpoint requires the ``permissions-debug`` permission. @@ -1180,11 +1238,20 @@ This endpoint requires the ``permissions-debug`` permission. Permission check view --------------------- -The ``/-/check`` endpoint evaluates a single action/resource pair and returns information indicating whether the access was allowed along with diagnostic information. +The ``/-/check`` endpoint evaluates and explains a single actor, action and resource decision. The explanation includes: + +* Every matching allow and deny rule, with its source and reason. +* The winning resource, parent or global scope. +* Rules ignored because a more specific rule matched, or because a deny won at the same scope. +* Actor restriction allowlists that included or excluded the resource. +* Additional actions required by the requested action. +* An explicit default-deny explanation when no rule matched. This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/check.json?action=view-instance``) to get the raw JSON response instead. -Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. +Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. The interactive form also accepts actor JSON, allowing a hypothetical actor to be tested without signing in as that actor. The JSON endpoint accepts the same value using the ``actor`` query string parameter. Use ``actor=null`` to represent an anonymous actor. + +This endpoint requires the ``permissions-debug`` permission. The hypothetical actor is used only for the decision being explained; access to the debug tool is checked against the actor who is actually signed in. .. _authentication_ds_actor: @@ -1386,6 +1453,16 @@ create-table Actor is allowed to create a database table. +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + +.. _actions_create_view: + +create-view +----------- + +Actor is allowed to create a database view. + ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) @@ -1408,18 +1485,6 @@ set-column-type Actor is allowed to set assigned :ref:`column types ` for columns in a table. -``resource`` - ``datasette.resources.TableResource(database, table)`` - ``database`` is the name of the database (string) - - ``table`` is the name of the table (string) - -.. _actions_set_label_columns: - -set-label-columns ------------------ - -Actor is allowed to set the :ref:`label column(s) ` used to build a display label for rows in a table. - ``resource`` - ``datasette.resources.TableResource(database, table)`` ``database`` is the name of the database (string) @@ -1437,6 +1502,18 @@ Actor is allowed to drop a database table. ``table`` is the name of the table (string) +.. _actions_drop_view: + +drop-view +--------- + +Actor is allowed to drop a database view. + +``resource`` - ``datasette.resources.TableResource(database, table)`` + ``database`` is the name of the database (string) + + ``table`` is the name of the view (string) + .. _actions_execute_sql: execute-sql diff --git a/docs/binary_data.rst b/docs/binary_data.rst index 0c890fe5..d41e36be 100644 --- a/docs/binary_data.rst +++ b/docs/binary_data.rst @@ -12,7 +12,12 @@ Datasette includes special handling for these binary values. The Datasette inter :width: 311px :alt: Screenshot showing download links next to binary data in the table view -Binary data is represented in ``.json`` exports using Base64 encoding. +.. _binary_json_format: + +Binary values in JSON +--------------------- + +Binary data is represented in ``.json`` exports using Base64 encoding. Datasette uses this representation for every ``BLOB`` value, including binary values that could also be decoded as UTF-8 text. https://latest.datasette.io/fixtures/binary_data.json?_shape=array @@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array } ] +The same format can be used with the :ref:`JSON write API `. +If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes: + +.. code-block:: json + + { + "data": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + +This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column. + +To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object: + +.. code-block:: json + + { + "data": { + "$raw": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + } + +``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again: + +.. code-block:: json + + { + "data": { + "$raw": { + "$raw": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + } + } + .. _binary_linking: Linking to binary downloads diff --git a/docs/changelog.rst b/docs/changelog.rst index ec32b57e..670166bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,77 @@ Changelog ========= +.. _v1_0_a37: + +1.0a37 (2026-07-14) +------------------- + +Performance improvement for SQL-backed permission checks, plus an improved permission debugging interface. + +- SQL used to resolve permission checks now aggregates permission rules before joining them to resources, improving performance on instances with large schemas. (:issue:`2832`) +- The :ref:`PermissionCheckView` permission debugger now explains why a decision was allowed or denied, including the matching rules. The interactive form can also test a hypothetical actor supplied as JSON, and the :ref:`permissions documentation ` now describes resolution rules in more detail. (:issue:`2841`) +- :ref:`db.execute_write(sql, ..., transaction=True) ` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) +- Refreshing a database's schema in Datasette's internal catalog is now performed as a single atomic operation. (:issue:`2831`) +- Fixed schema introspection, table pages, facets and table counts for tables with names containing a ``]`` character. Thanks, `TowyTowy `__. (:issue:`2431`, :pr:`2846`) +- ``/-/plugins.json`` once again returns a top-level JSON array of plugin objects, reverting the object envelope introduced in 1.0a36. This should fix a large number of trivial test failures in existing plugins. (:issue:`2842`, :pr:`2843`) + +.. _v1_0_a36: + +1.0a36 (2026-07-07) +------------------- + +The signature features of this alpha are new UIs for **inserting multiple rows at once** (from TSV, CSV or JSON) and for **creating a table from rows**, plus a large number of small **JSON API consistency fixes** in preparation for a 1.0 stable release. + +- Table pages now offer an "Insert multiple rows" mode in the row insertion dialog. This accepts pasted TSV, CSV or JSON, previews the parsed rows before inserting them, validates unknown columns as data is pasted and displays omitted auto integer primary keys as ``auto`` in the preview. (:pr:`2813`) +- The bulk insert UI can skip rows with existing primary keys, or update existing rows and insert new rows using the existing ``//
    {{ action.name }} {% if action.abbr %}{{ action.abbr }}{% endif %}
    {{ suggestion.database }} {{ suggestion.table }}{{ suggestion.label_columns|join(", ") }}{{ suggestion.label_column }}
    /-/upsert`` API when the actor has both :ref:`insert-row ` and :ref:`update-row ` permissions. (:pr:`2813`) +- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`) +- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text. (:issue:`2806`, :pr:`2822`) +- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. (:issue:`2806`, :pr:`2822`) +- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. +- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits. (:issue:`2823`) +- Row pages for tables with compound primary keys now return a ``400`` error instead of a ``500`` error when the URL row identifier does not contain the correct number of primary key values. Thanks, `Zain Dana Harper `__. (:issue:`2811`, :pr:`2815`) +- The :ref:`execute-write-sql ` interface now supports ``CREATE VIEW`` and ``DROP VIEW`` statements, gated by the new :ref:`create-view ` and :ref:`drop-view ` permissions. (:issue:`2819`, :pr:`2818`) +- Saved-query SQL analysis now handles recursive CTEs, fixing a bug where storing a valid read-only recursive query could be disabled by SQLite's internal ``SQLITE_RECURSIVE`` authorizer callback. (:issue:`2809`, :pr:`2812`) +- ``named_parameters()`` now correctly ignores SQLite comment markers that appear inside string literals, so query forms no longer drop later ``:named`` parameters from SQL such as ``select '--' || :name``. Thanks, `JSap0914 `__. (:pr:`2783`) +- Datasette's internal database schema is now managed using `sqlite-utils migrations `__, using the new dependency on ``sqlite-utils>=4.0``. (:issue:`2827`) +- ``datasette.utils.CustomJSONEncoder`` is now documented as a public API for plugins that need to serialize Datasette values to JSON. Thanks, `Chris Amico `__. (:issue:`1983`, :pr:`1996`) + +This release also includes the results of a `detailed consistency review `__ of Datasette's JSON API in preparation for the 1.0 stable release. Several of these changes are backwards-incompatible with previous 1.0 alphas. The new :ref:`API stability documentation ` describes exactly which parts of the JSON API are covered by the 1.0 stability promise. + +JSON API: breaking changes +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- JSON error responses now use a single canonical format across every endpoint: ``{"ok": false, "error": "...", "errors": [...], "status": 400}``. The ``error`` key joins all error messages together, ``errors`` is the full list of messages and ``status`` always matches the HTTP status code. The legacy ``title`` key is no longer included in JSON errors (it remains available to the HTML error template), and endpoints that previously returned bare ``{"error": ...}`` objects have been updated. See :ref:`json_api_errors`. +- Every JSON object success response now includes ``"ok": true``, including introspection endpoints such as ``/-/versions`` and ``/-/settings``. +- ``/-/plugins.json``, ``/-/databases.json`` and ``/-/actions.json`` now return objects - ``{"ok": true, "plugins": [...]}`` and equivalents - instead of top-level JSON arrays, so these responses can gain additional keys in the future without a breaking change. The ``datasette plugins`` CLI command still outputs a plain array. +- ``/-/databases`` now only lists databases the current actor is allowed to view. It previously listed every attached database, including their filesystem paths, to any actor with ``view-instance``. +- Requests with an invalid or expired ``Authorization: Bearer`` token now receive a ``401`` status with the standard error body and a ``WWW-Authenticate: Bearer error="invalid_token"`` header, instead of being silently treated as unauthenticated. Bearer tokens that no registered token handler recognizes are still ignored, so authentication plugins with their own token formats keep working. Plugin :ref:`token handlers ` can raise the new ``datasette.TokenInvalid`` exception to trigger the same behavior. +- Permission errors for JSON requests now return the standard JSON error format with a ``403`` status. The default forbidden handling previously rendered an HTML error page even for ``.json`` requests. +- ``POST`` to a write canned query now returns a ``400`` error when the SQL fails to execute, instead of a ``200`` status with ``"ok": false`` in the body. The error response includes the standard error keys plus a ``"redirect"`` key. +- The :ref:`row update API ` with ``"return": true`` now responds with a ``"rows"`` list, matching insert and upsert, instead of a singular ``"row"`` object. +- Row delete write failures - such as a constraint violation raised by a trigger - now return ``400`` instead of ``500``, matching the other write endpoints. +- ``//-/query.json`` with a missing or blank ``?sql=`` parameter now returns a ``400`` error, as the CSV format already did, instead of a ``200`` with empty rows. +- Unknown ``?_extra=`` names now return a ``400`` error for JSON and other data formats, instead of being silently ignored. HTML pages continue to ignore unknown names. +- Table JSON responses now include ``next_url`` alongside ``next`` by default - both are ``null`` on the final page. The now-redundant ``?_extra=next_url`` parameter has been removed. +- The stored query list JSON no longer includes ``has_more`` - ``"next": null`` is the end-of-results signal across the whole API. This change also uncovered and fixed a bug where the query list ``next_url`` pointed at the HTML page and was a relative path; it is now an absolute URL that preserves the requested format. +- Stored query JSON objects no longer duplicate the list of parameter names as both ``params`` and ``parameters`` - only ``parameters`` remains. The query create and update APIs no longer accept ``params`` as an input alias either; ``params`` is still the documented key for :ref:`queries defined in configuration `. +- Page size parameters are now consistent across the API: the stored query lists accept ``?_size=max`` and return a ``400`` error for values over the maximum instead of silently clamping them, and the ``/-/allowed`` and ``/-/rules`` permission debug endpoints renamed their ``page`` and ``page_size`` parameters to ``_page`` and ``_size``, matching the underscore grammar used by every other Datasette system parameter. +- ``/-/threads`` now requires the ``permissions-debug`` permission, since it exposes runtime internals such as file paths. It previously only required ``view-instance``. +- Trusted stored queries - those defined in configuration - can no longer be deleted through the JSON API or web interface, matching the existing restriction on editing them. +- The ``//-/schema`` endpoints now check the ``view-database`` permission before checking whether the database exists, so unauthorized actors can no longer probe for the existence of databases. +- SQL time limit errors in JSON responses are now a plain text message. The error string previously embedded an HTML fragment. +- The undocumented homepage JSON at ``/.json`` now returns ``databases`` as a list of objects rather than an object keyed by database name, matching every other collection in the API. +- The legacy ``.jsono`` format extension, long since superseded by ``?_shape=``, has been removed. + +JSON API: other improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The :ref:`write API ` endpoints now parse the request body as JSON regardless of the ``Content-Type`` header, so ``curl -d`` invocations work without remembering to set it. Invalid JSON is a ``400`` error. Cross-site request forgery remains prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` checks. This also fixes a ``500`` error from the insert API when the ``Content-Type`` header was missing entirely. +- New ``Response.error(messages, status=400)`` helper for plugins that need to return a JSON error in Datasette's standard format. See :ref:`internals_response`. +- New ``count_truncated`` extra for table JSON, included automatically whenever ``count`` is requested. ``true`` means the count reached Datasette's counting limit and the real number of rows may be higher. See :ref:`json_api_extra`. +- JSON endpoints that are not part of the documented stable API now declare themselves with an ``"unstable"`` key in their responses. +- New documentation covering the grammar for :ref:`boolean query string arguments `, the reason :ref:`upsert ` returns ``200`` where insert returns ``201``, and advice for plugin authors on :ref:`naming secret configuration keys ` so that ``/-/config`` redacts them automatically. + .. _v1_0_a35: 1.0a35 (2026-06-23) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7ca88c4e..2302f742 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -244,6 +244,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam custom query (default=1000) max_insert_rows Maximum rows that can be inserted at a time using the bulk insert API (default=100) + max_post_body_bytes Maximum size in bytes for a POST body read into + memory, e.g. JSON API requests - set 0 to disable + this limit (default=2097152) num_sql_threads Number of threads in the thread pool for executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds diff --git a/docs/configuration.rst b/docs/configuration.rst index 74f04225..d01f5153 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -936,50 +936,6 @@ You can override this automatic detection by specifying which column should be u } .. [[[end]]] -``label_column`` can also be set to a list of columns, in which case the values of those columns will be joined with a space to build the label: - -.. [[[cog - config_example(cog, textwrap.dedent( - """ - databases: - mydatabase: - tables: - example_table: - label_column: [first_name, last_name] - """).strip() - ) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - databases: - mydatabase: - tables: - example_table: - label_column: [first_name, last_name] - -.. tab:: datasette.json - - .. code-block:: json - - { - "databases": { - "mydatabase": { - "tables": { - "example_table": { - "label_column": [ - "first_name", - "last_name" - ] - } - } - } - } - } -.. [[[end]]] - .. _table_configuration_hidden: ``hidden`` diff --git a/docs/installation.rst b/docs/installation.rst index 33d3d6a1..ceec7f23 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -17,13 +17,6 @@ If you want to start making contributions to the Datasette project by installing Basic installation ================== -.. _installation_datasette_desktop: - -Datasette Desktop for Mac -------------------------- - -`Datasette Desktop `__ is a packaged Mac application which bundles Datasette together with Python and allows you to install and run Datasette directly on your laptop. This is the best option for local installation if you are not comfortable using the command line. - .. _installation_homebrew: Using Homebrew diff --git a/docs/internals.rst b/docs/internals.rst index c53f6832..d2bd46ef 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming ``.actor`` - dictionary (str -> Any) or None The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated. +``.max_post_body_bytes`` - integer + The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument. + The object also has the following awaitable methods: ``await request.form(files=False, ...)`` - FormData @@ -109,9 +112,11 @@ The object also has the following awaitable methods: ``await request.json()`` - Any Returns the parsed JSON body of a request submitted by ``POST``. -``await request.post_body()`` - bytes +``await request.post_body(max_bytes=None)`` - bytes Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data. + The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them. + And a class method that can be used to create fake request objects for use in tests: ``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)`` @@ -279,7 +284,7 @@ For example: content_type="application/xml; charset=utf-8", ) -The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods: +The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)``, ``Response.error(...)`` or ``Response.redirect(...)`` helper methods: .. code-block:: python @@ -290,6 +295,8 @@ The quickest way to create responses is using the ``Response.text(...)``, ``Resp text_response = Response.text( "This will become utf-8 encoded text" ) + # A JSON error in Datasette's standard error format: + error_response = Response.error("Cannot do that", 400) # Redirects are served as 302, unless you pass status=301: redirect_response = Response.redirect( "https://latest.datasette.io/" @@ -299,6 +306,8 @@ Each of these responses will use the correct corresponding content-type - ``text Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above. +``Response.error(messages, status=400)`` returns a JSON error in the :ref:`standard Datasette error format `. ``messages`` can be a single string or a list of strings. Use this for JSON-only endpoints; if your error should content-negotiate between JSON and HTML, raise ``Forbidden``, ``NotFound``, ``BadRequest`` or ``DatasetteError`` instead and Datasette's error handling will build the appropriate response. + .. _internals_response_asgi_send: Returning a response with .asgi_send(send) @@ -2014,8 +2023,8 @@ Example usage: .. _database_execute_write: -await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10) --------------------------------------------------------------------------------------------------------- +await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True) +-------------------------------------------------------------------------------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -2050,7 +2059,9 @@ If you need to retrieve every row returned by a statement, pass ``return_all=Tru If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. -Each call to ``execute_write()`` will be executed inside a transaction. +Each call to ``execute_write()`` will be executed inside a transaction. Pass +``transaction=False`` for statements such as ``VACUUM`` that cannot run inside +a transaction. .. _database_execute_write_script: @@ -2228,8 +2239,8 @@ The ``Database`` class also provides properties and methods for introspecting th ``await db.fts_table(table)`` - string or None The name of the FTS table associated with this table, if one exists. -``await db.label_columns_for_table(table)`` - list of strings - The label column(s) associated with this table - either automatically detected, set using the ``"label_column"`` key in configuration, or configured at runtime via the ``set-label-columns`` API, see :ref:`table_configuration_label_column`. Returns an empty list if no label column could be determined. +``await db.label_column_for_table(table)`` - string or None + The label column that is associated with this table - either automatically detected or using the ``"label_column"`` key in configuration, see :ref:`table_configuration_label_column`. ``await db.foreign_keys_for_table(table)`` - list of dictionaries Details of columns in this table which are foreign keys to other tables. A list of dictionaries where each dictionary is shaped like this: ``{"column": string, "other_table": string, "other_column": string}``. @@ -2354,6 +2365,14 @@ The internal database schema is as follows: .. code-block:: sql + CREATE TABLE "_sqlite_migrations" ( + "id" INTEGER PRIMARY KEY, + "migration_set" TEXT, + "name" TEXT, + "applied_at" TEXT + ); + CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name" + ON "_sqlite_migrations" ("migration_set", "name"); CREATE TABLE catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, diff --git a/docs/introspection.rst b/docs/introspection.rst index 4834f441..14b6249f 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -7,6 +7,10 @@ Datasette includes some pages and JSON API endpoints for introspecting the curre Each of these pages can be viewed in your browser. Add ``.json`` to the URL to get back the contents as JSON. +JSON responses that return an object include an ``"ok": true`` key, consistent with the rest of the :ref:`JSON API `. + +The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise `, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases. + .. _JsonDataView_metadata: /-/metadata @@ -37,6 +41,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example ` for this instance of Datasette. T .. code-block:: json { + "ok": true, "settings": { "template_debug": true, "trace_debug": true, @@ -129,20 +136,47 @@ Any keys that include the one of the following substrings in their names will be /-/databases ------------ -Shows currently attached databases. `Databases example `_: +Shows currently attached databases that the current actor is allowed to view, based on the ``view-database`` permission. `Databases example `_: .. code-block:: json - [ - { - "hash": null, - "is_memory": false, - "is_mutable": true, - "name": "fixtures", - "path": "fixtures.db", - "size": 225280 - } - ] + { + "ok": true, + "databases": [ + { + "hash": null, + "is_memory": false, + "is_mutable": true, + "name": "fixtures", + "path": "fixtures.db", + "size": 225280 + } + ] + } + +.. _JsonDataView_actions: + +/-/actions +---------- + +Shows all actions registered with the permission system, including those added by plugins. Requires the ``permissions-debug`` permission. + +.. code-block:: json + + { + "ok": true, + "actions": [ + { + "name": "view-instance", + "abbr": "vi", + "description": "View Datasette instance", + "takes_parent": false, + "takes_child": false, + "resource_class": null, + "also_requires": null + } + ] + } .. _JumpView: @@ -160,6 +194,7 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name. .. code-block:: json { + "ok": true, "matches": [ { "name": "fixtures", @@ -188,6 +223,7 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``: .. code-block:: json { + "ok": true, "matches": [ { "name": "fixtures: facetable", @@ -215,11 +251,12 @@ Without those query string arguments, the page lists up to five tables with dete /-/threads ---------- -Shows details of threads and ``asyncio`` tasks. `Threads example `_: +Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``permissions-debug`` permission, since it exposes runtime internals. `Threads example `_: .. code-block:: json { + "ok": true, "num_threads": 2, "threads": [ { @@ -251,6 +288,7 @@ Shows the currently authenticated actor. Useful for debugging Datasette authenti .. code-block:: json { + "ok": true, "actor": { "id": 1, "username": "some-user" diff --git a/docs/json_api.rst b/docs/json_api.rst index 5eb81b8c..a96fd73d 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -9,6 +9,53 @@ through the Datasette user interface can also be accessed as JSON via the API. To access the API for a page, either click on the ``.json`` link on that page or edit the URL and add a ``.json`` extension to it. +.. _json_api_stability: + +API stability +------------- + +Datasette 1.0 makes a stability promise for its JSON API: the endpoints, +parameters and response keys documented here and on the pages this +documentation links to will not change in backwards-incompatible ways for +the duration of the 1.x release series. + +Stability means: + +- Documented endpoints will keep their URLs, methods, parameters and + permission requirements. +- Documented response keys will keep their names and types. New keys may be + **added** in any release - clients should ignore keys they do not + recognize. +- The documented ``?_extra=`` names, ``?_shape=`` values and + :ref:`column filter operators ` are stable. +- Pagination tokens - the ``"next"`` key and ``?_next=`` parameter - are + **opaque strings**. Pass them back exactly as you received them; their + internal structure is not part of the API and can change at any time. +- The :ref:`standard error format ` and the + :ref:`API token format and restriction semantics ` are + stable, including the action abbreviations stored inside signed tokens. + +Some JSON endpoints are **exempt** from this promise: + +- Endpoints that are not documented include this marker key in their + responses and can change at any time:: + + "unstable": "This API is not part of Datasette's stable interface and may change at any time" + + This currently covers the instance homepage (``/.json``), the stored + query ``analyze``/``store``/``definition`` endpoints, ``/-/query/parameters``, + ``/-/execute-write/analyze`` and the JSON returned by the ``/-/permissions`` + debug playground. +- Debug and support endpoints are documented so you can use them, but their + JSON shapes are not frozen: :ref:`/-/threads `, + :ref:`/-/actions `, + the :ref:`permission debug endpoints ` + (``/-/allowed``, ``/-/rules``, ``/-/check``) and the + :ref:`table autocomplete endpoint `. +- Response keys explicitly labeled as unstable in this documentation, such + as the ``"analysis"`` block returned by :ref:`execute-write ` + and the ``debug`` and ``request`` extras. + .. _json_api_default: Default representation @@ -42,13 +89,49 @@ looks like this: "truncated": false } -``"ok"`` is always ``true`` if an error did not occur. +``"ok"`` is always ``true`` if an error did not occur. Every Datasette JSON endpoint that returns an object includes this key on success. The ``"rows"`` key is a list of objects, each one representing a row. The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). -For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. +For table pages, two additional keys are present: ``"next"``, an opaque token that can be used to retrieve the next page using ``?_next=TOKEN``, and ``"next_url"``, the full URL of that next page. Both are ``null`` on the final page. See :ref:`json_api_pagination`. + +.. _json_api_errors: + +Error responses +--------------- + +Every JSON error response from Datasette uses the same format: + +.. code-block:: json + + { + "ok": false, + "error": "Table not found", + "errors": [ + "Table not found" + ], + "status": 404 + } + +- ``"ok"`` is always ``false`` for an error. +- ``"errors"`` is a list of one or more error message strings. Endpoints that + validate multiple things at once - such as the :ref:`insert API ` - + may return several messages here. +- ``"error"`` is all of those messages joined with ``"; "``, for + convenience when displaying a single string. +- ``"status"`` matches the HTTP status code of the response. + +Some endpoints add extra context keys. For example, a SQL error from a +:ref:`custom query ` also includes the empty +``"rows"`` and ``"truncated"`` keys of the response it was unable to +produce. + +Permission errors use the same format: a request that fails a permission +check receives a ``403`` with this JSON error body when the URL ends in +``.json`` or the request sends an ``Accept: application/json`` or +``Content-Type: application/json`` header. .. _json_api_custom_sql: @@ -92,6 +175,7 @@ options: { "ok": true, "next": null, + "next_url": null, "rows": [ [3, "Detroit"], [2, "Los Angeles"], @@ -192,6 +276,10 @@ Here is an example Python function built using `requests `, for example ``{"ok": false, "error": "Unknown _extra: nope", ...}``. .. [[[cog from json_api_doc import table_extras @@ -266,6 +356,15 @@ The available table extras are listed below. 15 +``count_truncated`` + True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count. (May execute additional queries.) + + ``GET /fixtures/facetable.json?_extra=count,count_truncated`` + + .. code-block:: json + + false + ``count_sql`` SQL query string used to calculate the total count for the current table view, including active filters. @@ -338,17 +437,6 @@ The available table extras are listed below. "where state = \"CA\" sorted by pk" -``next_url`` - Full URL for the next page of results - - ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` - - ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. - - .. code-block:: json - - "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" - ``columns`` List of column names returned by this table, row or query. @@ -402,6 +490,25 @@ The available table extras are listed below. "pk" ] +``column_details`` + SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) + + ``GET /fixtures/binary_data.json?_size=0&_extra=column_details`` + + .. code-block:: json + + { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": false, + "default": null, + "is_pk": false, + "pk_position": 0, + "hidden": 0 + } + } + ``display_columns`` Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys. @@ -738,11 +845,11 @@ The available table extras are listed below. false ``expandable_columns`` - List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_columns)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_columns`` is a list of label column(s) in the referenced table, or ``None``. (See :ref:`expand_foreign_keys` for how to expand these labels.) + List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_column)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_column`` is the label column in the referenced table, or ``None``. (See :ref:`expand_foreign_keys` for how to expand these labels.) ``GET /fixtures/facetable.json?_extra=expandable_columns`` - Each item is a ``[foreign_key, label_columns]`` pair: the foreign key relationship, then the list of column(s) in the other table that would be used as the label for each expanded value. + Each item is a ``[foreign_key, label_column]`` pair: the foreign key relationship, then the column in the other table that would be used as the label for each expanded value. .. code-block:: json @@ -753,7 +860,7 @@ The available table extras are listed below. "other_table": "facet_cities", "other_column": "id" }, - ["name"] + "name" ] ] @@ -807,6 +914,25 @@ The following extras are available for row JSON responses. "id" ] +``column_details`` + SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) + + ``GET /fixtures/binary_data/1.json?_extra=column_details`` + + .. code-block:: json + + { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": false, + "default": null, + "is_pk": false, + "pk_position": 0, + "hidden": 0 + } + } + ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) @@ -1137,7 +1263,6 @@ The following extras are available for arbitrary SQL query responses and stored, "description_html": null, "hide_sql": false, "fragment": null, - "params": [], "parameters": [], "is_write": false, "is_private": false, @@ -1532,6 +1657,10 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +The request body is always parsed as JSON, regardless of the request's ``Content-Type`` header - a body that is not valid JSON returns a ``400`` error. Cross-site request forgery is prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` header checks rather than by content type requirements. + +The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data. + .. _ExecuteWriteView: Executing write SQL @@ -1565,7 +1694,7 @@ Unsupported SQL operations are rejected by default. ``VACUUM`` is not allowed in A successful response includes a message, the SQLite ``rowcount``, a ``"rows"`` list, a ``"truncated"`` flag and a summary of the operations that were executed: -The shape of the ``"analysis"`` block is not yet considered a stable API and may change in future Datasette releases. +The shape of the ``"analysis"`` block is not part of the :ref:`stable API ` and may change in future Datasette releases. .. code-block:: json @@ -1625,15 +1754,17 @@ the execute-write returning row limit, which defaults to 10: ] } -Errors use the standard Datasette error format: +Errors use the :ref:`standard Datasette error format `: .. code-block:: json { "ok": false, + "error": "Permission denied: need execute-write-sql", "errors": [ "Permission denied: need execute-write-sql" - ] + ], + "status": 403 } .. _TableInsertView: @@ -1660,6 +1791,8 @@ A single row can be inserted using the ``"row"`` key: } } +Column values can use the :ref:`binary value JSON format ` to write BLOB data. + If successful, this will return a ``201`` status code and the newly inserted row, for example: .. code-block:: json @@ -1727,9 +1860,11 @@ If any of your rows have a primary key that is already in use, you will get an e { "ok": false, + "error": "UNIQUE constraint failed: new_table.id", "errors": [ "UNIQUE constraint failed: new_table.id" - ] + ], + "status": 400 } Pass ``"ignore": true`` to ignore these errors and insert the other rows: @@ -1765,6 +1900,8 @@ An upsert is an insert or update operation. If a row with a matching primary key The upsert API is mostly the same shape as the :ref:`insert API `. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions. +It also accepts the same :ref:`binary value JSON format `. + :: POST //
    /-/upsert @@ -1802,7 +1939,7 @@ The above example will: Similar to ``/-/insert``, a ``row`` key with an object can be used instead of a ``rows`` array to upsert a single row. -If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. +If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. This is deliberately different from the ``201`` returned by :ref:`insert `: an upsert may update existing rows without creating anything, so it does not claim resource creation. Add ``"return": true`` to the request body to return full copies of the affected rows after they have been inserted or updated: @@ -1859,9 +1996,11 @@ When using upsert you must provide the primary key column (or columns if the tab { "ok": false, + "error": "Row 0 is missing primary key column(s): \"id\"", "errors": [ "Row 0 is missing primary key column(s): \"id\"" - ] + ], + "status": 400 } If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. @@ -1895,6 +2034,8 @@ To update a row, make a ``POST`` to ``//
    //-/update``. You only need to pass the columns you want to update. Any other columns will be left unchanged. +Updated values can use the :ref:`binary value JSON format `. + If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. Add ``"return": true`` to the request body to return the updated row: @@ -1914,14 +2055,16 @@ The returned JSON will look like this: { "ok": true, - "row": { - "id": 1, - "title": "New title", - "other_column": "Will be present here too" - } + "rows": [ + { + "id": 1, + "title": "New title", + "other_column": "Will be present here too" + } + ] } -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. @@ -1942,7 +2085,7 @@ To delete a row, make a ``POST`` to ``//
    //-/delete``. If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableCreateView: @@ -2098,6 +2241,8 @@ Datasette will create a table with a schema that matches those rows and insert t "pk": "id" } +Example rows can use the :ref:`binary value JSON format `, allowing Datasette to infer ``BLOB`` columns. + Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions. The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: @@ -2122,9 +2267,11 @@ If you pass a row to the create endpoint with a primary key that already exists { "ok": false, + "error": "UNIQUE constraint failed: creatures.id", "errors": [ "UNIQUE constraint failed: creatures.id" - ] + ], + "status": 400 } You can avoid this error by passing the same ``"ignore": true`` or ``"replace": true`` options to the create endpoint as you can to the :ref:`insert endpoint `. @@ -2360,7 +2507,7 @@ A successful response returns the new schema and the previous schema. If the req "operations_applied": 11 } -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableSetColumnTypeView: @@ -2424,49 +2571,7 @@ To clear an existing column type assignment, set ``column_type`` to ``null``: This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. - -.. _TableSetLabelColumnsView: - -Setting label columns -~~~~~~~~~~~~~~~~~~~~~ - -To set the label column(s) for a table, make a ``POST`` to ``//
    /-/set-label-columns``. This requires the :ref:`actions_set_label_columns` permission. - -:: - - POST //
    /-/set-label-columns - Content-Type: application/json - Authorization: Bearer dstok_ - -.. code-block:: json - - { - "columns": ["first_name", "last_name"] - } - -This will return a ``200`` response like this: - -.. code-block:: json - - { - "ok": true, - "database": "data", - "table": "people", - "columns": ["first_name", "last_name"] - } - -To revert to Datasette's automatic label column detection, set ``columns`` to ``null``: - -.. code-block:: json - - { - "columns": null - } - -This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. See :ref:`table_configuration_label_column` for more about label columns. - -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. .. _TableDropView: @@ -2503,4 +2608,4 @@ If you pass the following POST body: Then the table will be dropped and a status ``200`` response of ``{"ok": true}`` will be returned. -Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Any errors will use the :ref:`standard error format `, with a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/docs/pages.rst b/docs/pages.rst index ce88be12..65c03a49 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -95,7 +95,7 @@ Use the :ref:`ExecuteWriteView` JSON API to execute writable SQL programmaticall Stored query browsers --------------------- -The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. +The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. The JSON versions accept ``?_size=`` (default 50, ``max`` for the :ref:`setting_max_returned_rows` limit) and a ``?_next=`` pagination token. These pages support search, pagination and filters for read-only or writable queries and private or public queries. Adding a ``.json`` extension to either URL returns the same list as JSON. @@ -169,11 +169,13 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi .. code-block:: json { + "ok": true, "schemas": [ { "database": "content", "schema": "create table posts ..." } + ] } .. _DatabaseSchemaView: @@ -181,11 +183,11 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi Database schema --------------- -Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"`` and ``"schema"`` keys. +Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"`` and ``"schema"`` keys. .. _TableSchemaView: Table schema ------------ -Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"``, ``"table"``, and ``"schema"`` keys. +Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"``, ``"table"``, and ``"schema"`` keys. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 81ef4acd..049cb292 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1685,6 +1685,8 @@ forbidden(datasette, request, message) Plugins can use this to customize how Datasette responds when a 403 Forbidden error occurs - usually because a page failed a permission check, see :ref:`authentication_permissions`. +Datasette's default behavior returns the :ref:`standard JSON error format ` with a 403 status when the request path ends in ``.json`` or the request has an ``Accept: application/json`` or ``Content-Type: application/json`` header; other requests get an HTML error page. + If a plugin hook wishes to react to the error, it should return a :ref:`Response object `. This example returns a redirect to a ``/-/login`` page: @@ -2544,6 +2546,10 @@ The default ``SignedTokenHandler`` uses itsdangerous signed tokens (``dstok_`` p async def verify_token(self, datasette, token): # Look up token in database, return actor dict or None + # if this handler does not recognize the token. Raise + # datasette.TokenInvalid for a token this handler + # recognizes but rejects (revoked, expired) - Datasette + # will respond with a 401 error. ... diff --git a/docs/plugins.rst b/docs/plugins.rst index d2b5c20a..d32a9fe6 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -459,6 +459,8 @@ Secret configuration values Some plugins may need configuration that should stay secret - API keys for example. There are two ways in which you can store secret configuration values. +The :ref:`/-/config ` introspection endpoint redacts the values of any configuration keys whose names contain one of these substrings: ``secret``, ``key``, ``password``, ``token``, ``hash`` or ``dsn``. Name your plugin's secret configuration keys accordingly - for example ``api_key`` or ``client_secret`` - so they are automatically redacted there. + **As environment variables**. If your secret lives in an environment variable that is available to the Datasette process, you can indicate that the configuration value should be read from that environment variable like so: .. [[[cog diff --git a/docs/settings.rst b/docs/settings.rst index 5cd49113..9c114e4a 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -125,6 +125,23 @@ You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_insert_rows 1000 +.. _setting_max_post_body_bytes: + +max_post_body_bytes +~~~~~~~~~~~~~~~~~~~ + +Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API `. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB). + +This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over. + +If you increase :ref:`setting_max_insert_rows` to support larger bulk inserts you may need to increase this limit as well:: + + datasette mydatabase.db --setting max_post_body_bytes 10485760 + +Set it to 0 to disable the limit entirely:: + + datasette mydatabase.db --setting max_post_body_bytes 0 + .. _setting_num_sql_threads: num_sql_threads diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 371348fb..4c6e4426 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -657,7 +657,7 @@ There are three options for specifying that you would like the response to your - Include ``?_json=1`` in the URL that you POST to - Include ``"_json": 1`` in your JSON body, or ``&_json=1`` in your form encoded body -The JSON response will look like this: +A successful JSON response will look like this: .. code-block:: json @@ -667,7 +667,21 @@ The JSON response will look like this: "redirect": "/data/add_name" } -The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. +If the SQL fails to execute - for example a constraint violation - the response uses the :ref:`standard error format ` with a ``400`` status, plus the ``"redirect"`` key from the query configuration: + +.. code-block:: json + + { + "ok": false, + "error": "UNIQUE constraint failed: docs.id", + "errors": [ + "UNIQUE constraint failed: docs.id" + ], + "status": 400, + "redirect": null + } + +The ``"message"``, ``"error"`` and ``"redirect"`` values here take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. .. _pagination: diff --git a/docs/template_context.rst b/docs/template_context.rst index 5c6b1567..e445b335 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -98,7 +98,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re The color assigned to the database ``database_page_data`` - ``dict`` - JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``. + JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``. ``editable`` - ``bool`` Boolean indicating if the database is editable @@ -329,7 +329,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. Pagination token for the next page, or None ``next_url`` - ``str`` - Full URL for the next page of results + Full URL for the next page of results, or None if there are no more pages. See :ref:`json_api_pagination`. ``ok`` - ``bool`` True if the data for this page was retrieved without errors @@ -389,7 +389,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. SQL definition for this table ``table_insert_ui`` - ``dict`` - Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys. + Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys. ``table_page_data`` - ``dict`` JSON data used by JavaScript on the table page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs, optional ``insertRow`` data and optional ``alterTable`` data. diff --git a/pyproject.toml b/pyproject.toml index 215b2cca..70496cdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", - "sqlite-utils>=3.30,<4.0", + "sqlite-utils>=4.0", "asyncinject>=0.7", "setuptools", "pip", diff --git a/tests/test_api.py b/tests/test_api.py index f57d0206..191d064a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS +from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode from datasette.utils.sqlite import sqlite_version from datasette.version import __version__ from .fixtures import make_app_client, EXPECTED_PLUGINS @@ -26,8 +27,9 @@ async def test_homepage(ds_client): "title", ] databases = data.get("databases") - assert databases.keys() == {"fixtures": 0}.keys() - d = databases["fixtures"] + assert isinstance(databases, list) + assert [d["name"] for d in databases] == ["fixtures"] + d = databases[0] assert d["name"] == "fixtures" assert isinstance(d["tables_count"], int) assert isinstance(len(d["tables_and_views_truncated"]), int) @@ -42,8 +44,7 @@ async def test_homepage_sort_by_relationships(ds_client): response = await ds_client.get("/.json?_sort=relationships") assert response.status_code == 200 tables = [ - t["name"] - for t in response.json()["databases"]["fixtures"]["tables_and_views_truncated"] + t["name"] for t in response.json()["databases"][0]["tables_and_views_truncated"] ] assert tables == [ "simple_primary_key", @@ -250,8 +251,10 @@ def test_no_files_uses_memory_database(app_client_no_files): response = app_client_no_files.get("/.json") assert response.status == 200 assert { - "databases": { - "_memory": { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "databases": [ + { "name": "_memory", "hash": None, "color": "a6c7b9", @@ -266,7 +269,7 @@ def test_no_files_uses_memory_database(app_client_no_files): "views_count": 0, "private": False, }, - }, + ], "metadata": {}, } == response.json # Try that SQL query @@ -323,20 +326,15 @@ def test_sql_time_limit(app_client_shorter_time_limit): "/fixtures/-/query.json?sql=select+sleep(0.5)", ) assert 400 == response.status + expected_message = ( + "SQL query took too long. The time limit is" + " controlled by the sql_time_limit_ms setting." + ) assert response.json == { "ok": False, - "error": ( - "

    SQL query took too long. The time limit is controlled by the\n" - 'sql_time_limit_ms\n' - "configuration option.

    \n" - '\n' - "" - ), + "error": expected_message, + "errors": [expected_message], "status": 400, - "title": "SQL Interrupted", } @@ -350,7 +348,7 @@ async def test_custom_sql_time_limit(ds_client): "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5", ) assert response.status_code == 400 - assert response.json()["title"] == "SQL Interrupted" + assert response.json()["error"].startswith("SQL query took too long.") @pytest.mark.asyncio @@ -371,6 +369,40 @@ async def test_row(ds_client): assert response.json()["rows"] == [{"id": 1, "content": "hello"}] +@pytest.mark.asyncio +@pytest.mark.parametrize("suffix", ("", ".json")) +@pytest.mark.parametrize( + "row_path", + ( + "a", # too few components for a two-column primary key + "a,b,c", # too many components for a two-column primary key + ), +) +async def test_row_pk_arity_mismatch_returns_400(ds_client, row_path, suffix): + # A row URL with the wrong number of comma-separated primary key + # components used to raise an uncaught sqlite3.ProgrammingError (HTTP 500) + # because the SQL had one bind placeholder per PK column but params were + # only bound for the supplied components. It should be a 400 instead, + # mirroring the existing guard in datasette/views/table.py. + response = await ds_client.get( + "/fixtures/compound_primary_key/{}{}".format(row_path, suffix) + ) + assert response.status_code == 400 + if suffix == ".json": + assert response.json()["ok"] is False + assert response.json()["status"] == 400 + + +@pytest.mark.asyncio +async def test_row_compound_pk_correct_arity(ds_client): + # The valid two-component URL still resolves the row. + response = await ds_client.get( + "/fixtures/compound_primary_key/a,b.json?_shape=objects" + ) + assert response.status_code == 200 + assert response.json()["rows"] == [{"pk1": "a", "pk2": "b", "content": "c"}] + + @pytest.mark.asyncio async def test_row_strange_table_name(ds_client): response = await ds_client.get( @@ -429,7 +461,7 @@ async def test_row_foreign_key_tables(ds_client): @pytest.mark.asyncio async def test_row_extras(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables,column_details" ) assert response.status_code == 200 data = response.json() @@ -446,6 +478,45 @@ async def test_row_extras(ds_client): "format": "json", } assert len(data["foreign_key_tables"]) == 5 + id_detail = data["column_details"]["id"] + assert id_detail["type"].lower() == "integer" + assert id_detail == { + "type": id_detail["type"], + "sqlite_type": "INTEGER", + "notnull": False, + "default": None, + "is_pk": True, + "pk_position": 1, + "hidden": 0, + } + content_detail = data["column_details"]["content"] + assert content_detail["type"].lower() == "text" + assert content_detail == { + "type": content_detail["type"], + "sqlite_type": "TEXT", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, + } + + +@pytest.mark.asyncio +async def test_column_details_extra_row_for_null_blob(ds_client): + response = await ds_client.get("/fixtures/binary_data/3.json?_extra=column_details") + assert response.status_code == 200 + data_detail = response.json()["column_details"]["data"] + assert data_detail["type"].lower() == "blob" + assert data_detail == { + "type": data_detail["type"], + "sqlite_type": "BLOB", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, + } @pytest.mark.asyncio @@ -507,7 +578,7 @@ async def test_row_extra_render_cell(): def test_databases_json(app_client_two_attached_databases_one_immutable): response = app_client_two_attached_databases_one_immutable.get("/-/databases.json") - databases = response.json + databases = response.json["databases"] assert 2 == len(databases) extra_database, fixtures_database = databases assert "extra database" == extra_database["name"] @@ -523,8 +594,12 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): @pytest.mark.asyncio async def test_threads_json(ds_client): - response = await ds_client.get("/-/threads.json") - expected_keys = {"threads", "num_threads"} + ds_client.ds.root_enabled = True + try: + response = await ds_client.get("/-/threads.json", actor={"id": "root"}) + finally: + ds_client.ds.root_enabled = False + expected_keys = {"ok", "threads", "num_threads"} if sys.version_info >= (3, 7, 0): expected_keys.update({"tasks", "num_tasks"}) data = response.json() @@ -577,7 +652,7 @@ async def test_actions_json(ds_client): try: ds_client.ds.root_enabled = True response = await ds_client.get("/-/actions.json", actor={"id": "root"}) - data = response.json() + data = response.json()["actions"] finally: ds_client.ds.root_enabled = original_root_enabled assert isinstance(data, list) @@ -609,6 +684,7 @@ async def test_actions_json(ds_client): async def test_settings_json(ds_client): response = await ds_client.get("/-/settings.json") assert response.json() == { + "ok": True, "default_page_size": 50, "default_facet_size": 30, "default_allow_sql": True, @@ -616,6 +692,7 @@ async def test_settings_json(ds_client): "facet_time_limit_ms": 200, "max_returned_rows": 100, "max_insert_rows": 100, + "max_post_body_bytes": 2 * 1024 * 1024, "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, @@ -677,7 +754,7 @@ def test_config_cache_size(app_client_larger_cache_size): def test_config_force_https_urls(): with make_app_client(settings={"force_https_urls": True}) as client: response = client.get( - "/fixtures/facetable.json?_size=3&_facet=state&_extra=next_url,suggested_facets" + "/fixtures/facetable.json?_size=3&_facet=state&_extra=suggested_facets" ) assert response.json["next_url"].startswith("https://") assert response.json["facet_results"]["results"]["state"]["results"][0][ @@ -772,7 +849,9 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 assert ["foo-bar", "foo", "fixtures"] == [ d["name"] - for d in app_client_conflicting_database_names.get("/-/databases.json").json + for d in app_client_conflicting_database_names.get("/-/databases.json").json[ + "databases" + ] ] for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): data = app_client_conflicting_database_names.get(path).json @@ -843,13 +922,45 @@ async def test_tilde_encoded_database_names(db_name): ds = Datasette() ds.add_memory_database(db_name) response = await ds.client.get("/.json") - assert db_name in response.json()["databases"].keys() - path = response.json()["databases"][db_name]["path"] + databases_by_name = {d["name"]: d for d in response.json()["databases"]} + assert db_name in databases_by_name + path = databases_by_name[db_name]["path"] # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 +@pytest.mark.asyncio +@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar")) +async def test_table_with_reserved_characters_in_name(table_name): + # Table names containing characters such as "]" that cannot be escaped + # using SQLite [bracket] quoting used to break schema introspection and + # the table page - https://github.com/simonw/datasette/issues/2431 + ds = Datasette() + db = ds.add_memory_database("test_reserved_table_names") + await db.execute_write( + "create table {} (id integer primary key, name text)".format( + escape_sqlite(table_name) + ) + ) + await db.execute_write( + "insert into {} (id, name) values (1, 'one')".format(escape_sqlite(table_name)) + ) + # Schema introspection (populate_schema_tables) must not crash: + db_response = await ds.client.get("/test_reserved_table_names.json") + assert db_response.status_code == 200 + tables = {t["name"]: t for t in db_response.json()["tables"]} + assert tables[table_name]["count"] == 1 + # And the table page itself must load and return the row: + table_response = await ds.client.get( + "/test_reserved_table_names/{}.json?_shape=array".format( + tilde_encode(table_name) + ) + ) + assert table_response.status_code == 200 + assert table_response.json() == [{"id": 1, "name": "one"}] + + @pytest.mark.asyncio @pytest.mark.parametrize( "config,expected", @@ -883,7 +994,7 @@ async def test_config_json(config, expected): "/-/config.json should return redacted configuration" ds = Datasette(config=config) response = await ds.client.get("/-/config.json") - assert response.json() == expected + assert response.json() == {"ok": True, **expected} @pytest.mark.asyncio @@ -979,7 +1090,7 @@ async def test_config_json(config, expected): async def test_upgrade_metadata(metadata, expected_config, expected_metadata): ds = Datasette(metadata=metadata) response = await ds.client.get("/-/config.json") - assert response.json() == expected_config + assert response.json() == {"ok": True, **expected_config} response2 = await ds.client.get("/-/metadata.json") assert response2.json() == expected_metadata diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 76797742..a803fbbc 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,11 +1,23 @@ from datasette.app import Datasette from datasette.events import RenameTableEvent -from datasette.utils import escape_sqlite, sqlite3 +from datasette.utils import error_body, escape_sqlite, sqlite3 from .utils import last_event import pytest import time +def assert_schema_contains(fragment, schema): + assert fragment in schema, "Expected schema to contain {!r}, got {!r}".format( + fragment, schema + ) + + +def assert_schema_not_contains(fragment, schema): + assert ( + fragment not in schema + ), "Expected schema not to contain {!r}, got {!r}".format(fragment, schema) + + @pytest.fixture def ds_write(tmp_path_factory): db_directory = tmp_path_factory.mktemp("dbs") @@ -50,6 +62,136 @@ def _insert_and_fetch_created(conn, table, insert_sql): ).fetchone() +BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"} +BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}' + + +@pytest.mark.asyncio +async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "binary_create", + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + "double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert_schema_contains('"data" BLOB', response.json()["schema"]) + assert_schema_contains('"literal" TEXT', response.json()["schema"]) + + rows = (await ds_write.get_database("data").execute(""" + select + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal, + typeof(double_raw) as double_raw_type, + double_raw + from binary_create + """)).dicts() + assert rows == [ + { + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + "double_raw_type": "text", + "double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + } + ] + + +@pytest.mark.asyncio +async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write): + token = write_token(ds_write) + db = ds_write.get_database("data") + await db.execute_write( + "create table binary_api (id integer primary key, data blob, literal text)" + ) + + insert_response = await ds_write.client.post( + "/data/binary_api/-/insert", + json={ + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + }, + headers=_headers(token), + ) + assert insert_response.status_code == 201 + assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE + + upsert_response = await ds_write.client.post( + "/data/binary_api/-/upsert", + json={ + "rows": [ + { + "id": 2, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + ] + }, + headers=_headers(token), + ) + assert upsert_response.status_code == 200 + assert upsert_response.json() == {"ok": True} + + update_response = await ds_write.client.post( + "/data/binary_api/1/-/update", + json={ + "update": { + "data": {"$base64": True, "encoded": "/wAB"}, + "literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "return": True, + }, + headers=_headers(token), + ) + assert update_response.status_code == 200 + assert update_response.json()["rows"][0]["data"] == { + "$base64": True, + "encoded": "/wAB", + } + + rows = (await db.execute(""" + select + id, + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal + from binary_api + order by id + """)).dicts() + assert rows == [ + { + "id": 1, + "data_type": "blob", + "data_hex": "FF0001", + "literal_type": "text", + "literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + }, + { + "id": 2, + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + }, + ] + + @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) @@ -180,6 +322,34 @@ async def test_insert_rows(ds_write, return_rows): assert response.json()["rows"] == actual_rows +@pytest.mark.asyncio +async def test_insert_rows_post_body_too_large(tmp_path_factory): + db_path = str(tmp_path_factory.mktemp("dbs") / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette([db_path], settings={"max_post_body_bytes": 100}) + ds.root_enabled = True + token = write_token(ds) + response = await ds.client.post( + "/data/docs/-/insert", + json={"rows": [{"title": "x" * 200}]}, + headers=_headers(token), + ) + assert response.status_code == 413 + assert response.json() == error_body( + ["Request body exceeded maximum size of 100 bytes"], 413 + ) + # A small body should still work + response2 = await ds.client.post( + "/data/docs/-/insert", + json={"row": {"title": "hi"}}, + headers=_headers(token), + ) + assert response2.status_code == 201 + ds.close() + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,input,special_case,expected_status,expected_errors", @@ -202,8 +372,8 @@ async def test_insert_rows(ds_write, return_rows): "/data/docs/-/insert", {"rows": [{"title": "Test"} for i in range(10)]}, "bad_token", - 403, - ["Permission denied"], + 401, + ["Invalid token signature"], ), ( "/data/docs/-/insert", @@ -214,13 +384,6 @@ async def test_insert_rows(ds_write, return_rows): "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ], ), - ( - "/data/docs/-/insert", - {}, - "invalid_content_type", - 400, - ["Invalid content-type, must be application/json"], - ), ( "/data/docs/-/insert", [], @@ -402,20 +565,17 @@ async def test_insert_or_upsert_row_errors( json=input, headers={ "Authorization": "Bearer {}".format(token), - "Content-Type": ( - "text/plain" - if special_case == "invalid_content_type" - else "application/json" - ), + "Content-Type": "application/json", }, ) - actor_response = ( - await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) - ).json() - assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( - token_permissions - ) + if special_case != "bad_token": + actor_response = ( + await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) + ).json() + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -788,7 +948,12 @@ async def test_update_row_invalid_key(ds_write): headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == {"ok": False, "errors": ["Invalid keys: bad_key"]} + assert response.json() == { + "ok": False, + "error": "Invalid keys: bad_key", + "errors": ["Invalid keys: bad_key"], + "status": 400, + } @pytest.mark.asyncio @@ -1041,7 +1206,9 @@ async def test_alter_table_foreign_key_operations(ds_write): assert response.status_code == 200, response.text data = response.json() assert data["operations_applied"] == 2 - assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + assert_schema_contains( + '"owner_id" INTEGER REFERENCES "owners"("id")', data["schema"] + ) response = await ds_write.client.post( "/data/docs/-/alter", @@ -1052,7 +1219,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + assert_schema_not_contains('"owner_id" INTEGER REFERENCES', data["schema"]) response = await ds_write.client.post( "/data/docs/-/alter", @@ -1076,7 +1243,9 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert "[owner_id] INTEGER REFERENCES [categories]([id])" in data["schema"] + assert_schema_contains( + '"owner_id" INTEGER REFERENCES "categories"("id")', data["schema"] + ) response = await ds_write.client.post( "/data/docs/-/alter", @@ -1085,7 +1254,7 @@ async def test_alter_table_foreign_key_operations(ds_write): ) assert response.status_code == 200, response.text data = response.json() - assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + assert_schema_not_contains('"owner_id" INTEGER REFERENCES', data["schema"]) @pytest.mark.asyncio @@ -1103,10 +1272,9 @@ async def test_alter_table_foreign_key_requires_fk_table_for_fk_column(ds_write) headers=_headers(write_token(ds_write, permissions=["at"])), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["operations.0.add_foreign_key.args: fk_column requires fk_table"], - } + assert response.json() == error_body( + ["operations.0.add_foreign_key.args: fk_column requires fk_table"], 400 + ) @pytest.mark.asyncio @@ -1130,10 +1298,9 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["Could not detect single primary key for table 'accounts'"], - } + assert response.json() == error_body( + ["Could not detect single primary key for table 'accounts'"], 400 + ) @pytest.mark.asyncio @@ -1199,10 +1366,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == { - "ok": False, - "errors": ["Permission denied: need alter-table"], - } + assert response.json() == error_body(["Permission denied: need alter-table"], 403) @pytest.mark.asyncio @@ -1313,10 +1477,7 @@ async def test_foreign_key_targets_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == { - "ok": False, - "errors": ["Permission denied: need create-table"], - } + assert response.json() == error_body(["Permission denied: need create-table"], 403) @pytest.mark.asyncio @@ -1339,10 +1500,7 @@ async def test_alter_table_permission_denied(ds_write): headers=_headers(token), ) assert response.status_code == 403 - assert response.json() == { - "ok": False, - "errors": ["Permission denied: need alter-table"], - } + assert response.json() == error_body(["Permission denied: need alter-table"], 403) @pytest.mark.asyncio @@ -1486,9 +1644,9 @@ async def test_update_row(ds_write, input, expected_errors, use_return): assert response.json()["ok"] is True if not use_return: - assert "row" not in response.json() + assert "rows" not in response.json() else: - returned_row = response.json()["row"] + returned_row = response.json()["rows"][0] assert returned_row["id"] == pk for k, v in input.items(): assert returned_row[k] == v @@ -1623,12 +1781,12 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/one", "table_api_url": "http://localhost/data/one.json", "schema": ( - "CREATE TABLE [one] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [title] TEXT,\n" - " [score] INTEGER,\n" - " [weight] FLOAT,\n" - " [thumbnail] BLOB\n" + 'CREATE TABLE "one" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "score" INTEGER,\n' + ' "weight" REAL,\n' + ' "thumbnail" BLOB\n' ")" ), }, @@ -1660,10 +1818,10 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/two", "table_api_url": "http://localhost/data/two.json", "schema": ( - "CREATE TABLE [two] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [title] TEXT,\n" - " [score] FLOAT\n" + 'CREATE TABLE "two" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "score" REAL\n' ")" ), "row_count": 2, @@ -1689,10 +1847,10 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/three", "table_api_url": "http://localhost/data/three.json", "schema": ( - "CREATE TABLE [three] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [title] TEXT,\n" - " [score] FLOAT\n" + 'CREATE TABLE "three" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "score" REAL\n' ")" ), "row_count": 1, @@ -1714,7 +1872,7 @@ async def test_drop_table(ds_write, scenario): "table": "four", "table_url": "http://localhost/data/four", "table_api_url": "http://localhost/data/four.json", - "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), + "schema": ('CREATE TABLE "four" (\n' ' "name" TEXT\n' ")"), "row_count": 1, }, ["create-table", "insert-rows"], @@ -1734,8 +1892,8 @@ async def test_drop_table(ds_write, scenario): "table_url": "http://localhost/data/five", "table_api_url": "http://localhost/data/five.json", "schema": ( - "CREATE TABLE [five] (\n [type] TEXT,\n [key] INTEGER,\n" - " [title] TEXT,\n PRIMARY KEY ([type], [key])\n)" + 'CREATE TABLE "five" (\n "type" TEXT,\n "key" INTEGER,\n' + ' "title" TEXT,\n PRIMARY KEY ("type", "key")\n)' ), "row_count": 1, }, @@ -2021,6 +2179,12 @@ async def test_create_table( ) assert response.status_code == expected_status data = response.json() + if expected_response.get("ok") is False: + # Error expectations list their messages; derive the canonical envelope + expected_response = error_body(expected_response["errors"], expected_status) + if isinstance(expected_response, dict) and "schema" in expected_response: + assert data.get("schema") == expected_response["schema"] + expected_response = dict(expected_response, schema=data.get("schema")) assert data == expected_response # Should have tracked the expected events events = ds_write._tracked_events @@ -2063,7 +2227,9 @@ async def test_create_table_with_foreign_key(ds_write): ) assert response.status_code == 201 data = response.json() - assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + assert_schema_contains( + '"owner_id" INTEGER REFERENCES "owners"("id")', data["schema"] + ) @pytest.mark.asyncio @@ -2218,13 +2384,12 @@ async def test_create_table_column_validation(ds_write, column, expected_error): ) if expected_error: assert response.status_code == 400 - assert response.json() == {"ok": False, "errors": [expected_error]} + assert response.json() == error_body([expected_error], 400) else: assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["Could not detect single primary key for table 'owners'"], - } + assert response.json() == error_body( + ["Could not detect single primary key for table 'owners'"], 400 + ) @pytest.mark.asyncio @@ -2262,10 +2427,9 @@ async def test_create_table_foreign_key_without_fk_column_requires_single_pk(ds_ headers=_headers(token), ) assert response.status_code == 400 - assert response.json() == { - "ok": False, - "errors": ["Could not detect single primary key for table 'accounts'"], - } + assert response.json() == error_body( + ["Could not detect single primary key for table 'accounts'"], 400 + ) @pytest.mark.asyncio @@ -2415,10 +2579,9 @@ async def test_create_table_error_if_pk_changed(ds_write): headers=_headers(token), ) assert second_response.status_code == 400 - assert second_response.json() == { - "ok": False, - "errors": ["pk cannot be changed for existing table"], - } + assert second_response.json() == error_body( + ["pk cannot be changed for existing table"], 400 + ) @pytest.mark.asyncio @@ -2442,10 +2605,9 @@ async def test_create_table_error_rows_twice_with_duplicates(ds_write): headers=_headers(token), ) assert second_response.status_code == 400 - assert second_response.json() == { - "ok": False, - "errors": ["UNIQUE constraint failed: test_create_twice.id"], - } + assert second_response.json() == error_body( + ["UNIQUE constraint failed: test_create_twice.id"], 400 + ) @pytest.mark.asyncio @@ -2468,6 +2630,8 @@ async def test_method_not_allowed(ds_write, path): assert response.json() == { "ok": False, "error": "Method not allowed", + "errors": ["Method not allowed"], + "status": 405, } @@ -2535,10 +2699,9 @@ async def test_create_using_alter_against_existing_table( ) if not has_alter_permission: assert response2.status_code == 403 - assert response2.json() == { - "ok": False, - "errors": ["Permission denied: need alter-table"], - } + assert response2.json() == error_body( + ["Permission denied: need alter-table"], 403 + ) else: assert response2.status_code == 201 diff --git a/tests/test_auth.py b/tests/test_auth.py index 5868a21c..d2913ecc 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -236,7 +236,9 @@ def test_auth_create_token( @pytest.mark.asyncio async def test_auth_create_token_not_allowed_for_tokens(ds_client): - ds_tok = ds_client.ds.sign({"a": "test", "token": "dstok"}, "token") + ds_tok = ds_client.ds.sign( + {"a": "test", "token": "dstok", "t": int(time.time())}, "token" + ) response = await ds_client.get( "/-/create-token", headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, @@ -294,7 +296,7 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): try: if should_work: data = response.json() - assert data.keys() == {"actor"} + assert data.keys() == {"ok", "actor"} actor = data["actor"] expected_keys = {"id", "token"} if scenario != "valid_unlimited_token": @@ -304,8 +306,16 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): assert actor["token"] == "dstok" if scenario != "valid_unlimited_token": assert isinstance(actor["token_expires"], int) + elif scenario == "no_token": + # No credentials presented - request proceeds as anonymous + assert response.json() == {"ok": True, "actor": None} else: - assert response.json() == {"actor": None} + # Invalid credentials presented - hard 401 + assert response.status_code == 401 + data = response.json() + assert data["ok"] is False + assert data["status"] == 401 + assert response.headers["www-authenticate"].startswith("Bearer") finally: ds_client.ds._settings["allow_signed_tokens"] = True @@ -337,10 +347,11 @@ def test_cli_create_token(app_client, expires): } if expires and expires > 0: expected_actor["token_expires"] = details["t"] + expires - assert response.json == {"actor": expected_actor} + assert response.json == {"ok": True, "actor": expected_actor} else: - expected_actor = None - assert response.json == {"actor": expected_actor} + # Expired token - hard 401 + assert response.status == 401 + assert response.json["ok"] is False @pytest.mark.asyncio diff --git a/tests/test_autocomplete.py b/tests/test_autocomplete.py index 76b9c902..194fcf01 100644 --- a/tests/test_autocomplete.py +++ b/tests/test_autocomplete.py @@ -25,13 +25,14 @@ async def test_autocomplete_single_pk_exact_match_and_label_order(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 2}, "label": "Longer non-label pk match"}, {"pks": {"id": 20}, "label": "2"}, {"pks": {"id": 21}, "label": "22"}, {"pks": {"id": 3}, "label": "A label containing 2"}, {"pks": {"id": 200}, "label": "A"}, - ] + ], } @@ -52,12 +53,12 @@ async def test_autocomplete_blank_q_returns_no_results(): response = await ds.client.get("/autocomplete_blank/people/-/autocomplete?q=") assert response.status_code == 200 - assert response.json() == {"rows": []} + assert response.json() == {"ok": True, "rows": []} response = await ds.client.get("/autocomplete_blank/people/-/autocomplete") assert response.status_code == 200 - assert response.json() == {"rows": []} + assert response.json() == {"ok": True, "rows": []} @pytest.mark.asyncio @@ -81,11 +82,12 @@ async def test_autocomplete_initial_returns_latest_rows(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 3}, "label": "Cleo"}, {"pks": {"id": 2}, "label": "Bob"}, {"pks": {"id": 1}, "label": "Alice"}, - ] + ], } response = await ds.client.get( @@ -94,11 +96,12 @@ async def test_autocomplete_initial_returns_latest_rows(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 3}, "label": "Cleo"}, {"pks": {"id": 2}, "label": "Bob"}, {"pks": {"id": 1}, "label": "Alice"}, - ] + ], } @@ -121,9 +124,10 @@ async def test_autocomplete_escapes_like_characters(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"id": 1}, "label": "100% real"}, - ] + ], } @@ -149,11 +153,12 @@ async def test_autocomplete_compound_pk_searches_all_pk_columns(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"country": "mx", "code": "ca"}, "label": "Campeche"}, {"pks": {"country": "us", "code": "ca"}, "label": "California"}, {"pks": {"country": "ca", "code": "bc"}, "label": "British Columbia"}, - ] + ], } @@ -184,9 +189,10 @@ async def test_autocomplete_primary_key_called_label(): assert response.status_code == 200 assert response.json() == { + "ok": True, "rows": [ {"pks": {"label": "abc"}, "label": "Display value"}, - ] + ], } @@ -246,8 +252,9 @@ async def test_autocomplete_timeout_uses_prefix_fallback(monkeypatch): assert timeout_was_simulated data = response.json() assert data == { + "ok": True, "rows": [ {"pks": {"id": f"item-1999{i:02d}"}, "label": f"name 1999{i:02d}"} for i in range(10) - ] + ], } diff --git a/tests/test_base_view.py b/tests/test_base_view.py index 2cd4d601..c1b0cf20 100644 --- a/tests/test_base_view.py +++ b/tests/test_base_view.py @@ -53,6 +53,8 @@ async def test_get_view(): assert json.loads(post_json_response.body) == { "ok": False, "error": "Method not allowed", + "errors": ["Method not allowed"], + "status": 405, } assert post_json_response.status == 405 diff --git a/tests/test_cli.py b/tests/test_cli.py index f86d6909..cbd8edad 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -385,7 +385,9 @@ def test_setting_boolean_validation_false_values(value): ) # Should be forbidden (setting is false) assert result.exit_code == 1, result.output - assert "Forbidden" in result.output + error = json.loads(result.output) + assert error["ok"] is False + assert error["status"] == 403 @pytest.mark.parametrize("value", ("on", "true", "1")) @@ -425,8 +427,9 @@ def test_setting_default_allow_sql(default_allow_sql): assert json.loads(result.output)["rows"][0] == {"21": 21} else: assert result.exit_code == 1, result.output - # This isn't JSON at the moment, maybe it should be though - assert "Forbidden" in result.output + error = json.loads(result.output) + assert error["ok"] is False + assert error["status"] == 403 def test_sql_errors_logged_to_stderr(): @@ -444,7 +447,7 @@ def test_serve_create(tmpdir): cli, [str(db_path), "--create", "--get", "/-/databases.json"] ) assert result.exit_code == 0, result.output - databases = json.loads(result.output) + databases = json.loads(result.output)["databases"] assert { "name": "does_not_exist_yet", "is_mutable": True, @@ -493,7 +496,7 @@ def test_serve_duplicate_database_names(tmpdir): conn.close() result = runner.invoke(cli, [db_1_path, db_2_path, "--get", "/-/databases.json"]) assert result.exit_code == 0, result.output - databases = json.loads(result.output) + databases = json.loads(result.output)["databases"] assert {db["name"] for db in databases} == {"db", "db_2"} @@ -585,7 +588,7 @@ def test_duplicate_database_files_error(tmpdir): cli, ["serve", other_db_path, str(config_dir), "--get", "/-/databases.json"] ) assert result4.exit_code == 0 - databases = json.loads(result4.output) + databases = json.loads(result4.output)["databases"] assert {db["name"] for db in databases} == {"other", "data"} # Test that multiple directories raise an error diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index dc852201..fe9416d6 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -95,7 +95,10 @@ def test_serve_with_get_and_token(): ], ) assert 0 == result2.exit_code, result2.output - assert json.loads(result2.output) == {"actor": {"id": "root", "token": "dstok"}} + assert json.loads(result2.output) == { + "ok": True, + "actor": {"id": "root", "token": "dstok"}, + } def test_serve_with_get_exit_code_for_error(): @@ -130,8 +133,9 @@ def test_serve_get_actor(): ) assert result.exit_code == 0 assert json.loads(result.output) == { + "ok": True, "actor": { "id": "root", "extra": "x", - } + }, } diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 45a9e7d1..cd308ec9 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -9,7 +9,7 @@ from datasette.column_types import ( ) from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.utils import sqlite3 +from datasette.utils import error_body, sqlite3 from datasette.utils import StartupError import markupsafe import pytest @@ -322,12 +322,6 @@ async def test_clear_column_type_api(ds_ct): "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ], ), - ( - {"column": "title", "column_type": {"type": "email"}}, - "invalid_content_type", - 400, - ["Invalid content-type, must be application/json"], - ), ( [], None, @@ -413,11 +407,7 @@ async def test_set_column_type_api_errors( kwargs = { "headers": { "Authorization": f"Bearer {token}", - "Content-Type": ( - "text/plain" - if special_case == "invalid_content_type" - else "application/json" - ), + "Content-Type": "application/json", } } if special_case == "invalid_json": @@ -426,7 +416,7 @@ async def test_set_column_type_api_errors( kwargs["json"] = body response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs) assert response.status_code == expected_status - assert response.json() == {"ok": False, "errors": expected_errors} + assert response.json() == error_body(expected_errors, expected_status) @pytest.mark.asyncio diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 0a9b30d8..636b17eb 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -109,9 +109,10 @@ def test_settings(config_dir_client): def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status - assert "hooray.py" in {p["name"] for p in response.json} - assert "non_py_file.txt" not in {p["name"] for p in response.json} - assert "mypy_cache" not in {p["name"] for p in response.json} + plugins = response.json + assert "hooray.py" in {p["name"] for p in plugins} + assert "non_py_file.txt" not in {p["name"] for p in plugins} + assert "mypy_cache" not in {p["name"] for p in plugins} def test_templates_and_plugin(config_dir_client): @@ -136,7 +137,7 @@ def test_static_directory_browsing_not_allowed(config_dir_client): def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status - databases = response.json + databases = response.json["databases"] assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): diff --git a/tests/test_docs.py b/tests/test_docs.py index 13b3a549..0bcb5e62 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -248,7 +248,7 @@ async def test_homepage(): async def test_actor_is_null(): ds = Datasette(memory=True) response = await ds.client.get("/-/actor.json") - assert response.json() == {"actor": None} + assert response.json() == {"ok": True, "actor": None} # -- end test_actor_is_null -- @@ -258,5 +258,5 @@ async def test_signed_cookie_actor(): ds = Datasette(memory=True) cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} response = await ds.client.get("/-/actor.json", cookies=cookies) - assert response.json() == {"actor": {"id": "root"}} + assert response.json() == {"ok": True, "actor": {"id": "root"}} # -- end test_signed_cookie_actor -- diff --git a/tests/test_error_shape.py b/tests/test_error_shape.py new file mode 100644 index 00000000..768814fd --- /dev/null +++ b/tests/test_error_shape.py @@ -0,0 +1,749 @@ +""" +Tests for the canonical JSON error shape. + +Every JSON error response from Datasette should use one shape: + + { + "ok": false, + "error": "", + "errors": ["", ...], + "status": + } + +Additional context keys (for example "rows" and "truncated" on SQL errors) +are permitted, but "ok", "error", "errors" and "status" must always be +present and the legacy "title" key must not be. + +https://github.com/simonw/datasette/issues - 1.0 API consistency +""" + +import pytest +import time +from datasette.app import Datasette +from datasette.utils import sqlite3 + + +def assert_canonical_error(response, expected_status): + assert response.status_code == expected_status + data = response.json() + assert data["ok"] is False + assert isinstance(data["error"], str) + assert data["error"] + assert isinstance(data["errors"], list) + assert data["errors"] + assert all(isinstance(message, str) for message in data["errors"]) + assert data["error"] == "; ".join(data["errors"]) + assert data["status"] == expected_status + assert "title" not in data + return data + + +@pytest.fixture +def ds_error_shape(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette([db_path]) + ds.root_enabled = True + yield ds + ds.close() + + +# Shape 1: the exception handler (handle_exception.py) + + +@pytest.mark.asyncio +async def test_not_found_error_shape(ds_client): + response = await ds_client.get("/fixtures/no_such_table.json") + assert_canonical_error(response, 404) + + +@pytest.mark.asyncio +async def test_datasette_error_with_title_omits_title_key(ds_client): + # DatasetteError(title="Invalid SQL") previously leaked a "title" key + response = await ds_client.get( + "/fixtures/-/query.json?sql=update+facetable+set+state+=+1" + ) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Statement must be a SELECT"] + + +# Shape 2: the _error() helper (views/base.py) - write API and friends + + +@pytest.mark.asyncio +async def test_write_api_validation_error_shape(ds_error_shape): + token = "dstok_{}".format( + ds_error_shape.sign( + {"a": "root", "token": "dstok", "t": 0}, + namespace="token", + ) + ) + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + json={"rows": [{"nope": 1}, {"also_nope": 2}]}, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + data = assert_canonical_error(response, 400) + # Multiple messages: errors keeps them all, error joins them + assert len(data["errors"]) == 2 + assert data["errors"][0].startswith("Row 0") + assert data["errors"][1].startswith("Row 1") + + +@pytest.mark.asyncio +async def test_write_api_permission_denied_shape(ds_error_shape): + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + json={"rows": [{"title": "hello"}]}, + headers={"Content-Type": "application/json"}, + ) + assert_canonical_error(response, 403) + + +# Shape 3: the JSON renderer (renderer.py) + + +@pytest.mark.asyncio +async def test_sql_error_shape_keeps_context_keys(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?sql=select+*+from+no_such_table" + ) + data = assert_canonical_error(response, 400) + # Renderer errors keep their context keys + assert data["rows"] == [] + assert "truncated" in data + + +@pytest.mark.asyncio +async def test_invalid_shape_error_shape(ds_client): + response = await ds_client.get("/fixtures/-/query.json?sql=select+1&_shape=bananas") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Invalid _shape: bananas"] + + +@pytest.mark.asyncio +async def test_shape_object_on_query_is_a_400_error(ds_client): + # Previously returned HTTP 200 with an ok: false body + response = await ds_client.get("/fixtures/-/query.json?sql=select+1&_shape=object") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["_shape=object is only available on tables"] + + +# Shape 4: bare {"error": ...} from the permission debug endpoints + + +@pytest.mark.asyncio +async def test_allowed_missing_action_error_shape(ds_client): + response = await ds_client.get("/-/allowed.json") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["action parameter is required"] + + +@pytest.mark.asyncio +async def test_allowed_unknown_action_error_shape(ds_client): + response = await ds_client.get("/-/allowed.json?action=no_such_action") + assert_canonical_error(response, 404) + + +@pytest.mark.asyncio +async def test_check_unknown_action_error_shape(ds_error_shape): + response = await ds_error_shape.client.get( + "/-/check.json?action=no_such_action", + actor={"id": "root"}, + ) + assert_canonical_error(response, 404) + + +@pytest.mark.asyncio +async def test_rules_missing_action_error_shape(ds_error_shape): + response = await ds_error_shape.client.get( + "/-/rules.json", + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["action parameter is required"] + + +# Other stragglers + + +@pytest.mark.asyncio +async def test_method_not_allowed_error_shape(ds_client): + response = await ds_client.post("/fixtures.json") + assert_canonical_error(response, 405) + + +@pytest.mark.asyncio +async def test_schema_unknown_database_error_shape(ds_client): + response = await ds_client.get("/no_such_db/-/schema.json") + assert_canonical_error(response, 404) + + +# Forbidden responses (the default forbidden() hook) + + +@pytest.fixture +def ds_forbidden(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette( + [db_path], + config={"databases": {"data": {"tables": {"docs": {"allow": {"id": "root"}}}}}}, + ) + ds.root_enabled = True + yield ds + ds.close() + + +@pytest.mark.asyncio +async def test_forbidden_json_path_returns_canonical_json(ds_forbidden): + response = await ds_forbidden.client.get("/data/docs.json") + data = assert_canonical_error(response, 403) + assert "permission" in data["error"].lower() + + +@pytest.mark.asyncio +async def test_forbidden_accept_json_returns_canonical_json(ds_forbidden): + response = await ds_forbidden.client.get( + "/data/docs", headers={"Accept": "application/json"} + ) + assert_canonical_error(response, 403) + + +@pytest.mark.asyncio +async def test_forbidden_html_path_still_returns_html(ds_forbidden): + response = await ds_forbidden.client.get("/data/docs") + assert response.status_code == 403 + assert response.headers["content-type"].startswith("text/html") + + +@pytest.mark.asyncio +async def test_forbidden_json_path_allowed_actor_still_works(ds_forbidden): + response = await ds_forbidden.client.get("/data/docs.json", actor={"id": "root"}) + assert response.status_code == 200 + assert response.json()["ok"] is True + + +# Write canned queries: SQL failures must not return HTTP 200 + + +@pytest.fixture +def ds_write_query(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette( + [db_path], + config={ + "databases": { + "data": { + "queries": { + "add_doc": { + "sql": ( + "insert into docs (id, title)" " values (:id, :title)" + ), + "write": True, + }, + "add_doc_custom_error": { + "sql": ( + "insert into docs (id, title)" " values (:id, :title)" + ), + "write": True, + "on_error_message": "Custom error message", + "on_error_redirect": "/data", + }, + } + } + } + }, + ) + yield ds + ds.close() + + +@pytest.mark.asyncio +async def test_write_query_success_returns_200(ds_write_query): + response = await ds_write_query.client.post( + "/data/add_doc", + json={"id": 1, "title": "One"}, + headers={"Accept": "application/json"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["message"] == "Query executed, 1 row affected" + assert data["redirect"] is None + + +@pytest.mark.asyncio +async def test_write_query_sql_failure_returns_400(ds_write_query): + for _ in range(2): + response = await ds_write_query.client.post( + "/data/add_doc", + json={"id": 1, "title": "One"}, + headers={"Accept": "application/json"}, + ) + data = assert_canonical_error(response, 400) + assert "UNIQUE constraint failed" in data["error"] + # The redirect context key from the canned query flow is preserved + assert data["redirect"] is None + + +@pytest.mark.asyncio +async def test_write_query_failure_uses_on_error_message_and_redirect( + ds_write_query, +): + for _ in range(2): + response = await ds_write_query.client.post( + "/data/add_doc_custom_error", + json={"id": 1, "title": "One"}, + headers={"Accept": "application/json"}, + ) + data = assert_canonical_error(response, 400) + assert data["error"] == "Custom error message" + assert data["redirect"] == "/data" + + +@pytest.mark.asyncio +async def test_write_query_forbidden_is_canonical_403(ds_write_query): + # An untrusted write query run by an actor without execute-write-sql + # raises Forbidden, handled by the forbidden() hook + await ds_write_query.invoke_startup() + await ds_write_query.add_query( + "data", + name="untrusted_add", + sql="insert into docs (id, title) values (:id, :title)", + is_write=True, + is_trusted=False, + source="user", + owner_id="someone", + ) + response = await ds_write_query.client.post( + "/data/untrusted_add", + json={"id": 5, "title": "Five"}, + headers={"Accept": "application/json"}, + actor={"id": "someone"}, + ) + assert_canonical_error(response, 403) + + +@pytest.mark.asyncio +async def test_write_query_rejected_operation_is_canonical_403(ds_write_query): + # A rejected operation (VACUUM) raises QueryWriteRejected, handled by + # the dedicated branch in QueryView.post - root has execute-write-sql + ds_write_query.root_enabled = True + await ds_write_query.invoke_startup() + await ds_write_query.add_query( + "data", + name="vacuum_it", + sql="vacuum", + is_write=True, + is_trusted=False, + source="user", + owner_id="root", + ) + response = await ds_write_query.client.post( + "/data/vacuum_it", + json={}, + headers={"Accept": "application/json"}, + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 403) + assert data["redirect"] is None + + +# Row delete write failures must be 400, matching row update + + +@pytest.mark.asyncio +async def test_row_delete_write_failure_is_400(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.execute("insert into docs (id, title) values (1, 'One')") + conn.execute( + "create trigger no_delete before delete on docs " + "begin select raise(abort, 'deletes are blocked'); end" + ) + conn.commit() + conn.close() + ds = Datasette([db_path]) + ds.root_enabled = True + try: + response = await ds.client.post( + "/data/docs/1/-/delete", + json={}, + headers={"Content-Type": "application/json"}, + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 400) + assert "deletes are blocked" in data["error"] + finally: + ds.close() + + +# Invalid bearer tokens must produce 401, not silent anonymous access + + +@pytest.mark.asyncio +async def test_expired_token_returns_401(ds_error_shape): + token = "dstok_{}".format( + ds_error_shape.sign( + {"a": "root", "t": int(time.time()) - 2000, "d": 1000}, + namespace="token", + ) + ) + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + ) + data = assert_canonical_error(response, 401) + assert "expired" in data["error"].lower() + assert response.headers["www-authenticate"].startswith("Bearer") + + +@pytest.mark.asyncio +async def test_bad_signature_token_returns_401(ds_error_shape): + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": "Bearer dstok_garbage"} + ) + assert_canonical_error(response, 401) + assert response.headers["www-authenticate"].startswith("Bearer") + + +@pytest.mark.asyncio +async def test_unrecognized_token_prefix_stays_anonymous(ds_error_shape): + # No registered handler claims this token - it might belong to a + # plugin's actor_from_request hook, so it must not hard-fail + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": "Bearer sometoken_abc"} + ) + assert response.status_code == 200 + assert response.json() == {"ok": True, "actor": None} + + +@pytest.mark.asyncio +async def test_valid_token_still_authenticates(ds_error_shape): + token = "dstok_{}".format( + ds_error_shape.sign( + {"a": "root", "t": int(time.time())}, + namespace="token", + ) + ) + response = await ds_error_shape.client.get( + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + ) + assert response.status_code == 200 + assert response.json()["actor"]["id"] == "root" + + +@pytest.mark.asyncio +async def test_bad_token_beats_valid_cookie(ds_error_shape): + # A malformed Authorization header is a hard error even if a valid + # ds_actor cookie is also present + response = await ds_error_shape.client.get( + "/-/actor.json", + headers={"Authorization": "Bearer dstok_garbage"}, + cookies={"ds_actor": ds_error_shape.client.actor_cookie({"id": "root"})}, + ) + assert_canonical_error(response, 401) + + +@pytest.mark.asyncio +async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.close() + ds = Datasette([db_path], settings={"allow_signed_tokens": False}) + try: + token = "dstok_{}".format( + ds.sign({"a": "root", "t": int(time.time())}, namespace="token") + ) + response = await ds.client.get( + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + ) + data = assert_canonical_error(response, 401) + assert "not enabled" in data["error"] + finally: + ds.close() + + +# GET /db/-/query without SQL: 400 for data formats, HTML editor stays 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/fixtures/-/query.json", + "/fixtures/-/query.json?sql=", + ), +) +async def test_query_json_without_sql_is_400(ds_client, path): + response = await ds_client.get(path) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["?sql= is required"] + + +@pytest.mark.asyncio +async def test_query_html_without_sql_is_still_the_editor(ds_client): + response = await ds_client.get("/fixtures/-/query") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + + +# Write API return:true responses use "rows" consistently + + +@pytest.mark.asyncio +async def test_row_update_return_uses_rows_list(ds_error_shape): + await ds_error_shape.client.post( + "/data/docs/-/insert", + json={"row": {"id": 1, "title": "One"}}, + headers={"Content-Type": "application/json"}, + actor={"id": "root"}, + ) + response = await ds_error_shape.client.post( + "/data/docs/1/-/update", + json={"update": {"title": "Updated"}, "return": True}, + headers={"Content-Type": "application/json"}, + actor={"id": "root"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert "row" not in data + assert data["rows"] == [{"id": 1, "title": "Updated"}] + + +# Schema endpoints: no existence oracle, no 500 on unknown database + + +@pytest.mark.asyncio +async def test_schema_endpoints_no_existence_oracle(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key)") + conn.close() + ds = Datasette([db_path], default_deny=True) + ds.root_enabled = True + try: + # An actor without view-database cannot distinguish an existing + # database from a missing one + denied_existing = await ds.client.get("/data/-/schema.json") + denied_missing = await ds.client.get("/nope/-/schema.json") + assert denied_existing.status_code == denied_missing.status_code == 403 + + # An authorized actor sees the real thing + root_existing = await ds.client.get("/data/-/schema.json", actor={"id": "root"}) + assert root_existing.status_code == 200 + root_missing = await ds.client.get("/nope/-/schema.json", actor={"id": "root"}) + assert root_missing.status_code == 404 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_table_schema_unknown_database_is_404_not_500(ds_client): + response = await ds_client.get("/no_such_db/some_table/-/schema.json") + assert_canonical_error(response, 404) + + +# Unknown _extra names are a 400, not silently ignored + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/fixtures/facetable.json?_extra=nope", + "/fixtures/facetable.json?_extra=count,nope", + "/fixtures/simple_primary_key/1.json?_extra=nope", + "/fixtures/-/query.json?sql=select+1&_extra=nope", + ), +) +async def test_unknown_extra_is_400(ds_client, path): + response = await ds_client.get(path) + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Unknown _extra: nope"] + + +@pytest.mark.asyncio +async def test_html_only_extra_via_json_is_400(ds_client): + # display_rows exists for the HTML view but is not part of the JSON API + response = await ds_client.get("/fixtures/facetable.json?_extra=display_rows") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["Unknown _extra: display_rows"] + + +@pytest.mark.asyncio +async def test_unknown_extra_ignored_on_html_pages(ds_client): + response = await ds_client.get("/fixtures/facetable?_extra=nope") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + + +# /-/threads exposes runtime internals and requires permissions-debug + + +@pytest.mark.asyncio +async def test_threads_requires_permissions_debug(ds_error_shape): + denied = await ds_error_shape.client.get("/-/threads.json") + assert_canonical_error(denied, 403) + allowed = await ds_error_shape.client.get("/-/threads.json", actor={"id": "root"}) + assert allowed.status_code == 200 + assert allowed.json()["ok"] is True + + +# _size is the one page-size parameter, with uniform validation + + +@pytest.mark.asyncio +async def test_query_list_size_supports_max_keyword(ds_client): + response = await ds_client.get("/fixtures/-/queries.json?_size=max") + assert response.status_code == 200 + # ds_client runs with max_returned_rows=100 + assert response.json()["limit"] == 100 + + +@pytest.mark.asyncio +async def test_query_list_size_rejects_out_of_range(ds_client): + response = await ds_client.get("/fixtures/-/queries.json?_size=5000") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["_size must be <= 100"] + + +@pytest.mark.asyncio +async def test_query_list_size_rejects_non_integer(ds_client): + response = await ds_client.get("/fixtures/-/queries.json?_size=bananas") + data = assert_canonical_error(response, 400) + assert data["errors"] == ["_size must be a positive integer"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ("allowed", "rules")) +async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint): + base = "/-/{}.json?action=view-instance".format(endpoint) + ok = await ds_error_shape.client.get( + base + "&_size=1&_page=1", actor={"id": "root"} + ) + assert ok.status_code == 200 + assert ok.json()["page_size"] == 1 + + max_size = await ds_error_shape.client.get( + base + "&_size=max", actor={"id": "root"} + ) + assert max_size.status_code == 200 + assert max_size.json()["page_size"] == 200 + + too_big = await ds_error_shape.client.get(base + "&_size=500", actor={"id": "root"}) + data = assert_canonical_error(too_big, 400) + assert data["errors"] == ["_size must be <= 200"] + + bad_page = await ds_error_shape.client.get(base + "&_page=0", actor={"id": "root"}) + data = assert_canonical_error(bad_page, 400) + assert data["errors"] == ["_page must be a positive integer"] + + +# Write endpoints parse the body as JSON regardless of Content-Type + + +@pytest.mark.asyncio +async def test_insert_works_without_content_type_header(ds_error_shape): + # Previously a 500 AttributeError + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + content='{"row": {"id": 1, "title": "One"}}', + actor={"id": "root"}, + ) + assert response.status_code == 201 + assert response.json()["rows"][0]["title"] == "One" + + +@pytest.mark.asyncio +async def test_insert_works_with_form_content_type(ds_error_shape): + # Previously 400 "Invalid content-type, must be application/json" + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + content='{"row": {"id": 2, "title": "Two"}}', + headers={"Content-Type": "application/x-www-form-urlencoded"}, + actor={"id": "root"}, + ) + assert response.status_code == 201 + + +@pytest.mark.asyncio +async def test_insert_form_encoded_body_is_invalid_json(ds_error_shape): + response = await ds_error_shape.client.post( + "/data/docs/-/insert", + content="title=Three", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + actor={"id": "root"}, + ) + data = assert_canonical_error(response, 400) + assert data["errors"][0].startswith("Invalid JSON:") + + +@pytest.mark.asyncio +async def test_alter_and_set_column_type_ignore_content_type(ds_error_shape): + alter = await ds_error_shape.client.post( + "/data/docs/-/alter", + content='{"operations": [{"op": "add_column", "args": {"name": "extra"}}]}', + actor={"id": "root"}, + ) + assert alter.status_code == 200, alter.text + sct = await ds_error_shape.client.post( + "/data/docs/-/set-column-type", + content='{"column": "title", "column_type": {"type": "textarea"}}', + actor={"id": "root"}, + ) + assert sct.status_code == 200, sct.text + + +# SQL Interrupted errors carry plain text in JSON, not an HTML fragment + + +@pytest.mark.asyncio +async def test_sql_interrupted_json_error_is_plain_text(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5" + ) + data = assert_canonical_error(response, 400) + assert "<" not in data["error"] + assert data["error"].startswith("SQL query took too long.") + + +@pytest.mark.asyncio +async def test_sql_interrupted_html_page_keeps_rich_error(ds_client): + response = await ds_client.get( + "/fixtures/-/query?sql=select+sleep(0.01)&_timelimit=5" + ) + assert response.status_code == 400 + assert "\/((\w+)\/(\w+))\.json<\/a> - Get rows for") @@ -1731,6 +1753,8 @@ async def test_permission_check_view_requires_debug_permission(): data = response.json() assert data["action"] == "view-instance" assert data["allowed"] is True + assert data["explanation"]["allowed"] is True + assert data["explanation"]["summary"] @pytest.mark.asyncio @@ -1756,6 +1780,211 @@ async def test_permission_check_view_query_actions(action): } +@pytest.mark.asyncio +async def test_permission_check_explains_specificity_for_hypothetical_actor(): + ds = Datasette( + config={ + "permissions": {"view-table": {"id": "alice"}}, + "databases": { + "analytics": { + "permissions": {"view-table": False}, + "tables": { + "public": {"permissions": {"view-table": {"id": "alice"}}} + }, + } + }, + } + ) + ds.root_enabled = True + await ds.invoke_startup() + + def path_for(child): + return "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": child, + "actor": json.dumps({"id": "alice"}), + } + ) + + public_response = await ds.client.get(path_for("public"), actor={"id": "root"}) + assert public_response.status_code == 200 + public = public_response.json() + assert public["actor"] == {"id": "alice"} + assert public["allowed"] is True + assert public["explanation"]["allowed"] is True + assert public["explanation"]["winning_scope"] == "resource" + public_rules = public["explanation"]["matched_rules"] + assert any( + rule["scope"] == "resource" and rule["effect"] == "allow" and rule["decisive"] + for rule in public_rules + ) + assert any( + rule["scope"] == "parent" + and rule["effect"] == "deny" + and rule["ignored_because"] == "A more specific rule matched" + for rule in public_rules + ) + + private_response = await ds.client.get(path_for("private"), actor={"id": "root"}) + assert private_response.status_code == 200 + private = private_response.json() + assert private["allowed"] is False + assert private["explanation"]["allowed"] is False + assert private["explanation"]["winning_scope"] == "parent" + assert private["explanation"]["summary"].startswith("Denied by a parent-level rule") + + +@pytest.mark.asyncio +async def test_permission_check_explains_deny_wins_at_same_scope(): + ds = Datasette(config={"permissions": {"view-table": {"id": "someone-else"}}}) + ds.root_enabled = True + await ds.invoke_startup() + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": "users", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + assert data["explanation"]["winning_scope"] == "global" + rules = data["explanation"]["matched_rules"] + assert any(rule["effect"] == "deny" and rule["decisive"] for rule in rules) + assert any( + rule["effect"] == "allow" + and rule["ignored_because"] == "A deny rule matched at the same scope" + for rule in rules + ) + + +@pytest.mark.asyncio +async def test_permission_check_explains_default_deny(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "insert-row", + "parent": "analytics", + "child": "users", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["allowed"] is False + assert explanation["matched_rules"] == [] + assert explanation["winning_scope"] is None + assert explanation["summary"] == ( + "Denied because no permission rule matched this actor and resource." + ) + + +@pytest.mark.asyncio +async def test_permission_check_explains_actor_restrictions(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + restricted_actor = { + "id": "alice", + "_r": {"r": {"analytics": {"public": ["vt"]}}}, + } + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": "private", + "actor": json.dumps(restricted_actor), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["rule_allowed"] is True + assert explanation["restriction_allowed"] is False + assert explanation["allowed"] is False + assert explanation["restrictions"] + assert any( + restriction["allowed"] is False for restriction in explanation["restrictions"] + ) + assert "actor's restrictions" in explanation["summary"] + + +@pytest.mark.asyncio +async def test_permission_check_explains_required_actions(): + from datasette import hookimpl + from datasette.permissions import PermissionSQL + + class StoreQueryPermissions: + @hookimpl + def permission_resources_sql(self, actor, action): + if not actor or actor.get("id") != "alice": + return None + if action == "store-query": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'alice can store queries' AS reason" + ) + if action == "execute-sql": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 0 AS allow, 'alice cannot execute SQL' AS reason" + ) + + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + ds.pm.register(StoreQueryPermissions(), name="store-query-test") + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "store-query", + "parent": "analytics", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["rule_allowed"] is True + assert explanation["required_actions"][0]["action"] == "execute-sql" + assert explanation["required_actions"][0]["allowed"] is False + assert explanation["summary"] == ( + "Denied because store-query also requires execute-sql, which was denied." + ) + + +@pytest.mark.asyncio +async def test_permission_check_hypothetical_actor_validation(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.get( + "/-/check.json?action=view-instance&actor=not-json", + actor={"id": "root"}, + ) + assert response.status_code == 400 + assert response.json()["error"].startswith("Invalid actor JSON:") + + response = await ds.client.get( + "/-/check.json?action=view-instance&actor=%5B%5D", + actor={"id": "root"}, + ) + assert response.status_code == 400 + assert response.json()["error"] == "actor must be a JSON object or null" + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ @@ -1799,3 +2028,35 @@ async def test_root_allow_block_with_table_restricted_actor(): actor=admin_actor, ) assert result is True + + +@pytest.mark.asyncio +async def test_databases_json_respects_view_database(tmp_path_factory): + # https://github.com/simonw/datasette - /-/databases should not list + # databases the actor is not allowed to view + db_directory = tmp_path_factory.mktemp("dbs") + from datasette.utils import sqlite3 as _sqlite3 + + paths = [] + for name in ("public", "private"): + path = str(db_directory / "{}.db".format(name)) + conn = _sqlite3.connect(path) + conn.execute("vacuum") + conn.close() + paths.append(path) + ds = Datasette( + paths, + config={"databases": {"private": {"allow": {"id": "root"}}}}, + ) + ds.root_enabled = True + await ds.invoke_startup() + try: + anon_response = await ds.client.get("/-/databases.json") + assert anon_response.status_code == 200 + anon_names = {db["name"] for db in anon_response.json()["databases"]} + assert anon_names == {"public"} + root_response = await ds.client.get("/-/databases.json", actor={"id": "root"}) + root_names = {db["name"] for db in root_response.json()["databases"]} + assert root_names == {"public", "private"} + finally: + ds.close() diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 4466f976..eb1edb57 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -1,3 +1,4 @@ +import base64 import json import socket import subprocess @@ -10,6 +11,10 @@ import pytest from datasette.fixtures import write_fixture_database from datasette.utils.sqlite import sqlite3 +PNG_1X1_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=" +) + def find_free_port(): with socket.socket() as sock: @@ -17,7 +22,7 @@ def find_free_port(): return sock.getsockname()[1] -def wait_for_server(process, url, timeout=10): +def wait_for_server(process, url, timeout=30): deadline = time.monotonic() + timeout last_error = None while time.monotonic() < deadline: @@ -36,7 +41,20 @@ def wait_for_server(process, url, timeout=10): except httpx.HTTPError as ex: last_error = repr(ex) time.sleep(0.1) - raise AssertionError(f"Timed out waiting for {url}: {last_error}") + if process.poll() is None: + process.terminate() + try: + stdout, stderr = process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + else: + stdout, stderr = process.communicate() + raise AssertionError( + f"Timed out waiting for {url}: {last_error}\n" + f"stdout:\n{stdout}\n" + f"stderr:\n{stderr}" + ) @pytest.fixture @@ -102,6 +120,22 @@ def write_playwright_database(db_path): id integer primary key, created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) ); + create table binary_files ( + id integer primary key, + name text not null, + data blob + ); + create table bulk_defaults ( + id integer primary key, + title text not null, + status text not null default 'todo', + score integer default 5 + ); + create table upsert_items ( + id text primary key, + title text, + metadata text + ); insert into projects (title, metadata, logo, notes, score) values ( 'Build Datasette', @@ -110,7 +144,22 @@ def write_playwright_database(db_path): 'Initial notes', 5 ); + insert into upsert_items (id, title, metadata) values + ('existing', 'Existing title', '{"old": true}'); """) + conn.execute( + "insert into binary_files (name, data) values (?, ?)", + ("Raw bytes", b"\x00\x01\x02\x03"), + ) + conn.execute( + "insert into binary_files (name, data) values (?, ?)", + ("PNG image", PNG_1X1_BYTES), + ) + conn.execute( + "insert into binary_files (name, data) values (?, ?)", + ("Null bytes", None), + ) + conn.commit() finally: conn.close() @@ -123,6 +172,7 @@ def write_playwright_config(config_path): "data": { "permissions": { "create-table": True, + "insert-row": True, "set-column-type": True, }, "tables": { @@ -138,7 +188,6 @@ def write_playwright_config(config_path): "insert-row": True, "update-row": True, "delete-row": True, - "set-label-columns": True, }, }, "defaults_demo": { @@ -146,6 +195,25 @@ def write_playwright_config(config_path): "alter-table": True, }, }, + "binary_files": { + "label_column": "name", + "permissions": { + "insert-row": True, + "update-row": True, + "delete-row": True, + }, + }, + "bulk_defaults": { + "permissions": { + "insert-row": True, + }, + }, + "upsert_items": { + "permissions": { + "insert-row": True, + "update-row": True, + }, + }, }, }, }, @@ -279,6 +347,58 @@ def project_row(datasette_server, pk): return rows[0] +def binary_file_blob(datasette_server, pk): + response = httpx.get( + f"{datasette_server}data/binary_files/{pk}.blob", + params={"_blob_column": "data"}, + ) + response.raise_for_status() + return response.content + + +def wait_for_binary_control_file(binary_control, size, name=None): + binary_control.locator( + ".row-edit-binary-size", has_text=f"Binary: {size} bytes" + ).wait_for() + if name: + binary_control.locator(".row-edit-binary-name", has_text=name).wait_for() + + +def bulk_default_rows(datasette_server, **filters): + params = { + "_shape": "objects", + **{key: str(value) for key, value in filters.items()}, + } + response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params) + response.raise_for_status() + return response.json()["rows"] + + +def upsert_item_rows(datasette_server, **filters): + params = { + "_shape": "objects", + **{key: str(value) for key, value in filters.items()}, + } + response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params) + response.raise_for_status() + return response.json()["rows"] + + +def upsert_item_row(datasette_server, pk): + rows = upsert_item_rows(datasette_server, id=pk) + assert len(rows) == 1 + return rows[0] + + +def open_bulk_insert_dialog(page, url): + page.goto(url) + page.locator('button[data-table-action="insert-row"]').click() + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + return dialog + + def open_jump_menu(page): page.keyboard.press("/") page.locator("navigation-search .search-input").wait_for() @@ -382,6 +502,165 @@ def test_create_table_flow(page, datasette_server): assert "NOT NULL DEFAULT 'Untitled'" in schema +@pytest.mark.playwright +def test_create_table_from_data_flow(page, datasette_server): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + from_data = dialog.locator(".table-create-from-data") + assert from_data.inner_text() == "Create table from data" + from_data.click() + + assert dialog.locator(".table-create-columns").is_hidden() + assert dialog.locator(".table-create-data").is_visible() + assert dialog.locator(".table-create-save").inner_text() == "Preview rows" + assert ( + dialog.locator(".table-create-data-note").inner_text() + == "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea" + ) + assert dialog.locator(".table-create-data-open-file").inner_text() == "open a file" + assert ( + dialog.locator(".table-create-data-editor").evaluate( + """node => Array.from(node.children) + .filter((child) => !child.hidden) + .map((child) => child.className) + .join(" ")""" + ) + == ("table-create-data-note table-create-input table-create-data-textarea") + ) + assert dialog.locator(".table-create-manual").inner_text() == ( + "Create table manually" + ) + assert ( + dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id") + == "table-create-data-textarea" + ) + + textarea = dialog.locator(".table-create-data-textarea") + dropped_value = textarea.evaluate("""node => new Promise((resolve) => { + node.addEventListener("input", () => resolve(node.value), { once: true }); + const file = new File(["short_id,name\\nx,Ada"], "Repo Export 2026!!.CSV", { type: "text/csv" }); + const dataTransfer = new DataTransfer(); + dataTransfer.items.add(file); + node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer })); + })""") + assert dropped_value == "short_id,name\nx,Ada" + assert dialog.locator(".table-create-table-name").input_value() == ( + "repo_export_2026" + ) + + dialog.locator(".table-create-table-name").fill("playwright_from_data") + textarea.fill( + json.dumps( + { + "metadata": [{"short_id": "a", "name": "Ignored", "score": 9}], + "people": [ + {"short_id": "b", "name": "Ada", "score": 1}, + {"short_id": "c", "name": "Bob", "score": 2.5}, + ], + } + ) + ) + dialog.locator(".table-create-save").click() + + assert dialog.locator(".table-create-data-textarea").is_hidden() + assert dialog.locator(".table-create-save").inner_text() == "Create table" + assert dialog.locator(".table-create-cancel").inner_text() == "Back" + assert dialog.locator(".table-create-data-preview-summary").inner_text() == ( + "Previewing 2 rows." + ) + assert dialog.locator(".table-create-data-primary-key").input_value() == "short_id" + preview_text = dialog.locator(".table-create-data-preview-table").inner_text() + assert "short_id" in preview_text + assert "Ada" in preview_text + assert "2.5" in preview_text + preview_cell_style = dialog.locator( + ".table-create-data-preview-table td" + ).first.evaluate( + """node => ({ + overflowWrap: getComputedStyle(node).overflowWrap, + whiteSpace: getComputedStyle(node).whiteSpace + })""" + ) + assert preview_cell_style == { + "overflowWrap": "anywhere", + "whiteSpace": "normal", + } + + dialog.locator(".table-create-cancel").click() + assert dialog.evaluate("node => node.open") + assert dialog.locator(".table-create-data-textarea").is_visible() + assert '"people"' in dialog.locator(".table-create-data-textarea").input_value() + assert dialog.locator(".table-create-save").inner_text() == "Preview rows" + + dialog.locator(".table-create-save").click() + assert dialog.locator(".table-create-save").inner_text() == "Create table" + dialog.locator(".table-create-save").click() + page.wait_for_url("**/data/playwright_from_data") + + response = httpx.get( + f"{datasette_server}data/playwright_from_data.json?_shape=objects" + ) + response.raise_for_status() + data = response.json() + assert data["rows"] == [ + {"short_id": "b", "name": "Ada", "score": 1}, + {"short_id": "c", "name": "Bob", "score": 2.5}, + ] + + +@pytest.mark.playwright +def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank( + page, datasette_server +): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + dialog.locator(".table-create-from-data").click() + dialog.locator(".table-create-table-name").fill("playwright_numeric_blanks") + dialog.locator(".table-create-data-textarea").fill("name,score\nA,1\nB,") + dialog.locator(".table-create-save").click() + + assert dialog.locator(".table-create-save").inner_text() == "Create table" + preview_text = dialog.locator(".table-create-data-preview-table").inner_text() + assert "A" in preview_text + assert "1" in preview_text + assert "B" in preview_text + assert "null" in preview_text + + dialog.locator(".table-create-save").click() + page.wait_for_url("**/data/playwright_numeric_blanks") + + response = httpx.get( + f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects" + ) + response.raise_for_status() + assert response.json()["rows"] == [ + {"name": "A", "score": 1}, + {"name": "B", "score": None}, + ] + + schema_response = httpx.get( + f"{datasette_server}data/-/query.json", + params={ + "sql": ( + "select type from pragma_table_info('playwright_numeric_blanks') " + "where name = 'score'" + ) + }, + ) + schema_response.raise_for_status() + assert schema_response.json()["rows"] == [{"type": "INTEGER"}] + + @pytest.mark.playwright def test_create_table_foreign_key_selection_updates_column_type(page, datasette_server): page.goto(f"{datasette_server}data") @@ -802,11 +1081,128 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins( @pytest.mark.playwright def test_insert_row_flow_uses_custom_column_field(page, datasette_server): + page.add_init_script(""" + (() => { + let clipboardText = ""; + Object.defineProperty(navigator, "clipboard", { + configurable: true, + get: () => ({ + writeText: async (text) => { + clipboardText = String(text); + }, + readText: async () => clipboardText, + }), + }); + })(); + """) page.goto(f"{datasette_server}data/projects") page.locator('button[data-table-action="insert-row"]').click() dialog = page.locator("#row-edit-dialog") dialog.wait_for() + bulk_insert = dialog.locator(".row-edit-bulk-insert") + bulk_insert.wait_for() + assert bulk_insert.inner_text() == "Insert multiple rows" + current_url = page.url + bulk_insert.click() + assert page.url == current_url + assert dialog.evaluate("node => node.open") + assert dialog.locator(".row-edit-fields").is_hidden() + assert dialog.locator(".row-edit-bulk").is_visible() + assert dialog.locator(".row-edit-save").inner_text() == "Preview rows" + assert ( + dialog.locator(".row-edit-bulk-note").inner_text() + == "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea" + ) + assert dialog.locator(".row-edit-bulk-open-file").inner_text() == "open a file" + assert ( + dialog.locator(".row-edit-bulk-editor").evaluate( + """node => Array.from(node.children) + .filter((child) => !child.hidden) + .map((child) => child.className) + .join(" ")""" + ) + == ( + "row-edit-bulk-note row-edit-input row-edit-bulk-textarea " + "row-edit-bulk-actions" + ) + ) + copy_template = dialog.locator(".row-edit-copy-template") + assert copy_template.inner_text() == "Copy spreadsheet template" + assert ( + dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id") + == "row-edit-bulk-textarea" + ) + assert dialog.locator(".row-edit-copy-template-label-narrow").text_content() == ( + "Copy template" + ) + assert dialog.locator(".row-edit-bulk-template-note").inner_text() == ( + "You can paste the template into Google Sheets or Excel." + ) + assert dialog.locator(".row-edit-bulk-template-note-narrow").text_content() == ( + "Paste into Google Sheets or Excel" + ) + copy_template.click() + page.wait_for_function( + """() => document.querySelector(".row-edit-copy-template").textContent === "Copied" """ + ) + assert page.evaluate("navigator.clipboard.readText()") == ( + "title\tmetadata\tlogo\tnotes\tscore" + ) + textarea = dialog.locator(".row-edit-bulk-textarea") + textarea.fill("title\tmetadata\nFrom TSV\t{}") + assert textarea.input_value() == "title\tmetadata\nFrom TSV\t{}" + dropped_value = textarea.evaluate("""node => new Promise((resolve) => { + node.addEventListener("input", () => resolve(node.value), { once: true }); + const file = new File(["\\n\\ntitle,metadata\\nFrom CSV,{}\\n,\\n , \\n"], "rows.csv", { type: "text/csv" }); + const dataTransfer = new DataTransfer(); + dataTransfer.items.add(file); + node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer })); + })""") + assert dropped_value == "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n" + dialog.locator(".row-edit-save").click() + assert dialog.evaluate("node => node.open") + assert dialog.locator(".row-edit-bulk-textarea").is_hidden() + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == ( + "Previewing 1 row." + ) + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "id" in preview_text + assert "title" in preview_text + assert "metadata" in preview_text + assert "From CSV" in preview_text + assert dialog.locator(".row-edit-bulk-preview-auto").first.text_content() == "auto" + assert "null" not in preview_text + assert "undefined" not in preview_text + preview_cell_style = dialog.locator( + ".row-edit-bulk-preview-table td" + ).first.evaluate( + """node => ({ + overflowWrap: getComputedStyle(node).overflowWrap, + whiteSpace: getComputedStyle(node).whiteSpace + })""" + ) + assert preview_cell_style == { + "overflowWrap": "anywhere", + "whiteSpace": "normal", + } + assert dialog.locator(".row-edit-cancel").inner_text() == "Back" + dialog.locator(".row-edit-cancel").click() + assert dialog.evaluate("node => node.open") + assert dialog.locator(".row-edit-bulk-textarea").is_visible() + assert dialog.locator(".row-edit-bulk-textarea").input_value() == ( + "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n" + ) + assert dialog.locator(".row-edit-save").inner_text() == "Preview rows" + single_insert = dialog.locator(".row-edit-single-insert") + assert single_insert.inner_text() == "Insert single row" + single_insert.click() + assert dialog.locator(".row-edit-bulk").is_hidden() + assert dialog.locator(".row-edit-fields").is_visible() + assert dialog.locator(".row-edit-save").inner_text() == "Insert row" dialog.locator('input[name="title"]').fill("Launch Datasette Cloud") dialog.locator('textarea[name="metadata"]').fill( '{"ok": false, "source": "playwright"}' @@ -836,6 +1232,206 @@ def test_insert_row_flow_uses_custom_column_field(page, datasette_server): assert data["score"] == 5 +@pytest.mark.playwright +def test_bulk_insert_preview_inserts_rows(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + dialog.locator(".row-edit-bulk-textarea").fill( + json.dumps( + { + "metadata": [{"title": "Ignored", "metadata": "{}"}], + "projects": [ + {"title": "Bulk one", "metadata": "{}"}, + {"title": "Bulk two", "metadata": "{}"}, + ], + } + ) + ) + dialog.locator(".row-edit-save").click() + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + dialog.locator(".row-edit-save").click() + dialog.locator( + ".row-edit-bulk-progress-status", has_text="2 rows inserted." + ).wait_for() + assert dialog.locator(".row-edit-cancel").inner_text() == "Close and view table" + dialog.locator(".row-edit-cancel").click() + page.wait_for_load_state("domcontentloaded") + assert page.url == f"{datasette_server}data/projects" + + assert project_rows(datasette_server, title="Bulk one") + assert project_rows(datasette_server, title="Bulk two") + + +@pytest.mark.playwright +def test_bulk_insert_upsert_option_updates_existing_and_inserts_new( + page, datasette_server +): + dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/upsert_items") + textarea = dialog.locator(".row-edit-bulk-textarea") + conflict_field = dialog.locator(".row-edit-bulk-conflict") + conflict_select = dialog.locator(".row-edit-bulk-conflict-mode") + + assert conflict_field.is_hidden() + textarea.fill("title\nNo primary key") + assert conflict_field.is_hidden() + + textarea.fill( + "id,title,metadata\nexisting,Updated by upsert,{}\nnew,Inserted by upsert,{}" + ) + assert conflict_field.is_visible() + assert ( + dialog.locator(".row-edit-bulk-conflict-label").inner_text() + == "If the row exists already" + ) + assert conflict_select.input_value() == "ignore" + assert ( + conflict_select.evaluate("""node => Array.from(node.options) + .filter((option) => !option.hidden) + .map((option) => option.textContent.trim())""") + == [ + "Stop with an error", + "Skip existing rows", + "Update existing and insert new", + ] + ) + + conflict_select.select_option("upsert") + assert conflict_select.input_value() == "upsert" + dialog.locator(".row-edit-save").click() + + assert dialog.locator(".row-edit-save").inner_text() == "Update or insert rows" + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "Updated by upsert" in preview_text + assert "Inserted by upsert" in preview_text + + dialog.locator(".row-edit-save").click() + dialog.locator( + ".row-edit-bulk-progress-status", has_text="2 rows upserted." + ).wait_for() + + assert upsert_item_row(datasette_server, "existing")["title"] == ( + "Updated by upsert" + ) + assert upsert_item_row(datasette_server, "new")["title"] == "Inserted by upsert" + + +@pytest.mark.playwright +def test_bulk_insert_conflicts_hide_upsert_without_update_permission( + page, datasette_server +): + dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/bulk_defaults") + textarea = dialog.locator(".row-edit-bulk-textarea") + conflict_field = dialog.locator(".row-edit-bulk-conflict") + conflict_select = dialog.locator(".row-edit-bulk-conflict-mode") + + textarea.fill("title\nOnly title") + assert conflict_field.is_hidden() + + textarea.fill("id,title\n1,Only title") + assert conflict_field.is_visible() + assert conflict_select.input_value() == "ignore" + assert ( + conflict_select.evaluate("""node => Array.from(node.options) + .filter((option) => !option.hidden) + .map((option) => option.textContent.trim())""") + == [ + "Stop with an error", + "Skip existing rows", + ] + ) + assert conflict_select.locator('option[value="upsert"]').evaluate( + "node => node.hidden && node.disabled" + ) + + +@pytest.mark.playwright +def test_bulk_insert_live_validation_reports_unknown_columns(page, datasette_server): + dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/projects") + textarea = dialog.locator(".row-edit-bulk-textarea") + error = dialog.locator(".row-edit-error") + save = dialog.locator(".row-edit-save") + + textarea.fill(json.dumps([{"id2": 1, "title": "Unknown column"}])) + error.wait_for() + assert error.inner_text() == "JSON row 1 has unknown column id2." + assert save.is_disabled() + assert textarea.evaluate("node => document.activeElement === node") + + textarea.fill("id2,title\n1,Unknown column") + assert error.inner_text() == "Unknown column id2 in header row." + assert save.is_disabled() + + textarea.fill("[") + assert error.is_hidden() + assert not save.is_disabled() + + textarea.fill(json.dumps([{"id": 1, "title": "Known column"}])) + assert error.is_hidden() + assert not save.is_disabled() + assert dialog.locator(".row-edit-bulk-conflict").is_visible() + assert dialog.locator(".row-edit-bulk-conflict-mode").input_value() == "ignore" + + +@pytest.mark.playwright +def test_bulk_insert_omits_columns_absent_from_pasted_input(page, datasette_server): + page.goto(f"{datasette_server}data/bulk_defaults") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + dialog.locator(".row-edit-bulk-textarea").fill("title\nOnly title") + dialog.locator(".row-edit-save").click() + + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "Only title" in preview_text + assert "undefined" not in preview_text + assert dialog.locator(".row-edit-bulk-preview-auto").inner_text() == "auto" + + dialog.locator(".row-edit-save").click() + dialog.locator( + ".row-edit-bulk-progress-status", has_text="1 row inserted." + ).wait_for() + + rows = bulk_default_rows(datasette_server, title="Only title") + assert rows == [ + { + "id": 1, + "title": "Only title", + "status": "todo", + "score": 5, + } + ] + + +@pytest.mark.playwright +def test_bulk_insert_preview_accepts_single_column_input(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + dialog.locator(".row-edit-bulk-textarea").fill("title\none\ntwo\nthree") + dialog.locator(".row-edit-save").click() + + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == ( + "Previewing 3 rows." + ) + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "one" in preview_text + assert "two" in preview_text + assert "three" in preview_text + assert "null" not in preview_text + assert "undefined" not in preview_text + + @pytest.mark.playwright def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): page.goto(f"{datasette_server}data/projects") @@ -843,6 +1439,9 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): dialog = page.locator("#row-edit-dialog") dialog.wait_for() + assert dialog.locator(".row-edit-bulk-insert").is_hidden() + assert dialog.locator(".row-edit-single-insert").is_hidden() + assert dialog.locator(".row-edit-bulk").is_hidden() title = dialog.locator('input[name="title"]') title.wait_for() title.fill("Build Datasette, edited") @@ -878,6 +1477,120 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): assert data["notes"] == "Edited from Playwright" +@pytest.mark.playwright +def test_edit_row_binary_control_shows_size_and_image_preview(page, datasette_server): + page.goto(f"{datasette_server}data/binary_files") + page.locator('tr[data-row="1"] button[data-row-action="edit"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + raw_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + raw_control.wait_for() + assert ( + raw_control.locator(".row-edit-binary-size").inner_text() == "Binary: 4 bytes" + ) + assert raw_control.locator(".row-edit-binary-preview img").count() == 0 + assert dialog.locator('textarea[name="data"]').count() == 0 + assert dialog.locator('input[type="hidden"][name="data"]').count() == 1 + + dialog.locator(".row-edit-cancel").click() + page.locator('tr[data-row="2"] button[data-row-action="edit"]').click() + + image_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + image_control.wait_for() + assert ( + image_control.locator(".row-edit-binary-size").inner_text() + == f"Binary: {len(PNG_1X1_BYTES)} bytes" + ) + image = image_control.locator(".row-edit-binary-preview img") + image.wait_for() + assert image.get_attribute("src").startswith("blob:") + + +@pytest.mark.playwright +def test_edit_row_binary_control_replaces_blob_from_file(page, datasette_server): + replacement = b"Replacement \x00 bytes" + + page.goto(f"{datasette_server}data/binary_files") + page.locator('tr[data-row="1"] button[data-row-action="edit"]').click() + + dialog = page.locator("#row-edit-dialog") + binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + binary_control.wait_for() + binary_control.locator('input[type="file"]').set_input_files( + { + "name": "replacement.bin", + "mimeType": "application/octet-stream", + "buffer": replacement, + } + ) + wait_for_binary_control_file(binary_control, len(replacement), "replacement.bin") + + dialog.locator(".row-edit-save").click() + page.locator(".row-mutation-status", has_text="Updated row 1").wait_for() + assert binary_file_blob(datasette_server, 1) == replacement + + +@pytest.mark.playwright +def test_edit_row_binary_control_handles_null_blob(page, datasette_server): + replacement = b"From NULL" + + page.goto(f"{datasette_server}data/binary_files") + page.locator('tr[data-row="3"] button[data-row-action="edit"]').click() + + dialog = page.locator("#row-edit-dialog") + binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + binary_control.wait_for() + assert ( + binary_control.locator(".row-edit-binary-size").inner_text() == "No binary data" + ) + binary_control.locator('input[type="file"]').set_input_files( + { + "name": "from-null.bin", + "mimeType": "application/octet-stream", + "buffer": replacement, + } + ) + wait_for_binary_control_file(binary_control, len(replacement), "from-null.bin") + + dialog.locator(".row-edit-save").click() + page.locator(".row-mutation-status", has_text="Updated row 3").wait_for() + assert binary_file_blob(datasette_server, 3) == replacement + + +@pytest.mark.playwright +def test_insert_row_binary_control_accepts_pasted_file(page, datasette_server): + pasted = b"Pasted \x00 bytes" + + page.goto(f"{datasette_server}data/binary_files") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator('input[name="name"]').fill("Pasted bytes") + binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + binary_control.wait_for() + binary_control.evaluate( + """(node, bytes) => { + const transfer = new DataTransfer(); + transfer.items.add( + new File([new Uint8Array(bytes)], "pasted.bin", { + type: "application/octet-stream" + }) + ); + const event = new Event("paste", { bubbles: true, cancelable: true }); + Object.defineProperty(event, "clipboardData", { value: transfer }); + node.dispatchEvent(event); + }""", + list(pasted), + ) + wait_for_binary_control_file(binary_control, len(pasted), "pasted.bin") + + dialog.locator(".row-edit-save").click() + page.locator(".row-mutation-status", has_text="Inserted row 4").wait_for() + assert binary_file_blob(datasette_server, 4) == pasted + + @pytest.mark.playwright def test_delete_row_flow_removes_row(page, datasette_server): page.goto(f"{datasette_server}data/projects") @@ -891,45 +1604,3 @@ def test_delete_row_flow_removes_row(page, datasette_server): page.locator(".row-mutation-status", has_text="Deleted row 1").wait_for() page.locator('tr[data-row="1"]').wait_for(state="detached") assert project_rows(datasette_server, id=1) == [] - - -@pytest.mark.playwright -def test_set_label_columns_flow(page, datasette_server): - page.goto(f"{datasette_server}data/projects") - page.locator("details.actions-menu-links summary").click() - page.locator('button[data-table-action="set-label-columns"]').click() - - dialog = page.locator("#table-label-columns-dialog") - dialog.wait_for() - assert ( - dialog.locator(".modal-title").inner_text() - == "Set label column(s) for projects" - ) - assert dialog.locator(".table-label-columns-reset").is_enabled() - - def checked_column_names(): - return dialog.locator( - ".table-label-columns-row:has(.table-label-columns-checkbox:checked) " - ".table-label-columns-name" - ).all_inner_texts() - - assert checked_column_names() == ["title"] - - notes_row = dialog.locator(".table-label-columns-row", has_text="notes") - notes_row.locator(".table-label-columns-checkbox").check() - dialog.locator(".table-label-columns-save").click() - dialog.wait_for(state="hidden") - - page.locator("details.actions-menu-links summary").click() - page.locator('button[data-table-action="set-label-columns"]').click() - dialog.wait_for() - assert checked_column_names() == ["title", "notes"] - - dialog.locator(".table-label-columns-reset").click() - dialog.wait_for(state="hidden") - - page.locator("details.actions-menu-links summary").click() - page.locator('button[data-table-action="set-label-columns"]').click() - dialog.wait_for() - assert checked_column_names() == ["title"] - assert dialog.locator(".table-label-columns-reset").is_disabled() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index da14c714..59b1c0bf 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -783,9 +783,13 @@ async def test_hook_permission_resources_sql(): @pytest.mark.asyncio async def test_actor_json(ds_client): - assert (await ds_client.get("/-/actor.json")).json() == {"actor": None} + assert (await ds_client.get("/-/actor.json")).json() == { + "ok": True, + "actor": None, + } assert (await ds_client.get("/-/actor.json?_bot2=1")).json() == { - "actor": {"id": "bot2", "1+1": 2} + "ok": True, + "actor": {"id": "bot2", "1+1": 2}, } diff --git a/tests/test_queries.py b/tests/test_queries.py index 6dfcc8b7..ffa948a9 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -8,6 +8,7 @@ from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage +from datasette.utils import UNSTABLE_API_MESSAGE from datasette.utils.asgi import Forbidden from datasette.utils.sqlite import sqlite3, supports_returning @@ -879,7 +880,7 @@ async def test_query_list_html_defaults_to_twenty_and_shows_pagination(): assert response.text.count('aria-label="Query pagination"') == 1 assert "Demo query 20" in response.text assert "Demo query 21" not in response.text - assert 'href="/data/-/queries?_next=' in response.text + assert 'href="http://localhost/data/-/queries?_next=' in response.text assert len(json_response.json()["queries"]) == 25 @@ -1125,6 +1126,47 @@ async def test_query_update_api_rejects_config_only_fields(): assert query.on_success_message_sql is None +@pytest.mark.asyncio +async def test_query_api_rejects_params_alias(): + # "params" is a datasette.yaml configuration key, not an API input - + # the API only accepts "parameters" + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_params_alias", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + store_response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + "params": ["name"], + } + }, + ) + assert store_response.status_code == 400 + assert store_response.json()["errors"] == ["Invalid keys: params"] + assert await ds.get_query("data", "by_name") is None + + await ds.add_query( + "data", + "editable", + "select * from dogs where name = :name", + source="user", + owner_id="root", + ) + update_response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={"update": {"params": ["name"]}}, + ) + assert update_response.status_code == 400 + assert update_response.json()["errors"] == ["Invalid keys: params"] + + @pytest.mark.asyncio async def test_query_update_api_rejects_trusted_queries_but_internal_update_allowed(): ds = Datasette( @@ -2140,7 +2182,11 @@ async def test_query_parameters_endpoint_uses_get_sql_only(): ) assert response.status_code == 200 - assert response.json() == {"ok": True, "parameters": ["name", "id"]} + assert response.json() == { + "ok": True, + "unstable": UNSTABLE_API_MESSAGE, + "parameters": ["name", "id"], + } assert permission_denied_response.status_code == 403 assert permission_denied_response.json()["errors"] == [ "Permission denied: need execute-sql" @@ -3093,9 +3139,9 @@ async def test_untrusted_stored_write_query_rejects_virtual_table_control_insert ) assert denied_response.status_code == 403 - assert denied_response.json()["message"] == ( + assert denied_response.json()["errors"] == [ "Writes to virtual tables are not allowed in user-supplied SQL" - ) + ] assert ( await db.execute("select count(*) from docs where docs match 'hello'") ).first()[0] == 1 @@ -3214,6 +3260,74 @@ async def test_execute_write_create_table_uses_create_table_permission(): assert not await db.table_exists("should_not_exist") +@pytest.mark.asyncio +async def test_execute_write_create_view_uses_create_view_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "permissions": { + "insert-row": {"id": "row-writer"}, + "update-row": {"id": "row-writer"}, + }, + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["creator", "row-writer"]}, + "execute-write-sql": {"id": ["creator", "row-writer"]}, + "create-view": {"id": "creator"}, + } + } + }, + }, + ) + db = ds.add_memory_database("execute_write_create_view", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + analysis_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "creator"}, + params={"sql": "create view dog_names as select id, name from dogs"}, + ) + allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "creator"}, + json={"sql": "create view dog_names as select id, name from dogs"}, + ) + row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "create view should_not_exist as select id from dogs"}, + ) + + assert analysis_response.status_code == 200 + analysis_data = analysis_response.json() + assert analysis_data["ok"] is True + assert analysis_data["execute_disabled"] is False + assert analysis_data["analysis_rows"] == [ + { + "operation": "create", + "database": "data", + "table": "dog_names", + "required_permission": "create-view", + "source": None, + "allowed": True, + } + ] + + assert allowed_response.status_code == 200 + assert allowed_response.json()["ok"] is True + assert allowed_response.json()["message"] == "Query executed" + assert await db.view_exists("dog_names") + + assert row_permission_response.status_code == 403 + assert row_permission_response.json()["errors"] == [ + "Permission denied: need create-view on data" + ] + assert not await db.view_exists("should_not_exist") + + @pytest.mark.parametrize( ( "database_name", @@ -3264,8 +3378,20 @@ async def test_execute_write_create_table_uses_create_table_permission(): (), "drop-table", ), + ( + "execute_write_drop_view", + "dropper", + "drop view dogs_view", + "drop view cats_view", + "Permission denied: need drop-view on data/cats_view", + ( + "create view dogs_view as select * from dogs", + "create view cats_view as select * from cats", + ), + "drop-view", + ), ), - ids=("alter-table", "create-index", "drop-index", "drop-table"), + ids=("alter-table", "create-index", "drop-index", "drop-table", "drop-view"), ) @pytest.mark.asyncio async def test_execute_write_schema_operations_use_schema_permissions( @@ -3300,7 +3426,12 @@ async def test_execute_write_schema_operations_use_schema_permissions( "drop-table": {"id": "dropper"}, "view-table": {"id": "alterer"}, } - } + }, + "dogs_view": { + "permissions": { + "drop-view": {"id": "dropper"}, + } + }, }, } }, @@ -3353,6 +3484,9 @@ async def test_execute_write_schema_operations_use_schema_permissions( elif expected_state == "drop-table": assert not await db.table_exists("dogs") assert await db.table_exists("cats") + elif expected_state == "drop-view": + assert not await db.view_exists("dogs_view") + assert await db.view_exists("cats_view") @pytest.mark.asyncio @@ -3654,3 +3788,81 @@ async def test_stored_write_query_with_truncated_returning_message(): assert response.status_code == 200 assert response.json()["ok"] is True assert response.json()["message"] == "Query executed" + + +@pytest.mark.asyncio +async def test_query_delete_api_rejects_trusted_queries(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "editor"}, + "delete-query": {"id": "editor"}, + }, + "queries": { + "trusted_report": { + "sql": "select 1 as one", + }, + }, + } + } + }, + ) + ds.add_memory_database("query_delete_trusted_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/trusted_report/-/delete", + actor={"id": "editor"}, + json={}, + ) + assert response.status_code == 403 + assert response.json()["errors"] == [ + "Trusted queries cannot be deleted using the API" + ] + # The query must still exist + assert await ds.get_query("data", "trusted_report") is not None + + # The HTML confirmation page refuses too + get_response = await ds.client.get( + "/data/trusted_report/-/delete", + actor={"id": "editor"}, + ) + assert get_response.status_code == 403 + + # datasette.remove_query() remains available for internal use + await ds.remove_query("data", "trusted_report") + assert await ds.get_query("data", "trusted_report") is None + + +@pytest.mark.asyncio +async def test_stored_query_json_uses_parameters_not_params(): + ds = Datasette( + memory=True, + config={ + "databases": { + "data": { + "queries": { + "with_params": { + "sql": "select :name as name, :age as age", + "params": ["name", "age"], + }, + }, + } + } + }, + ) + ds.add_memory_database("query_parameters_key", name="data") + await ds.invoke_startup() + + definition = (await ds.client.get("/data/with_params/-/definition")).json() + assert definition["query"]["parameters"] == ["name", "age"] + assert "params" not in definition["query"] + + listing = (await ds.client.get("/data/-/queries.json")).json() + query = [q for q in listing["queries"] if q["name"] == "with_params"][0] + assert query["parameters"] == ["name", "age"] + assert "params" not in query diff --git a/tests/test_routes.py b/tests/test_routes.py index 24c702fc..4965ce76 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -73,7 +73,7 @@ async def ds_with_route(): @pytest.mark.asyncio async def test_db_with_route_databases(ds_with_route): response = await ds_with_route.client.get("/-/databases.json") - assert response.json()[0] == { + assert response.json()["databases"][0] == { "name": "original-name", "route": "custom-route-name", "path": None, diff --git a/tests/test_success_envelope.py b/tests/test_success_envelope.py new file mode 100644 index 00000000..46a78ef0 --- /dev/null +++ b/tests/test_success_envelope.py @@ -0,0 +1,169 @@ +""" +Tests for the canonical JSON success envelope. + +Every JSON object returned by a Datasette endpoint on success should include +"ok": true. /-/plugins intentionally returns a top-level array instead, while +/-/databases and /-/actions use the object envelope. +""" + +import pytest +from datasette.app import Datasette +from datasette.utils import sqlite3, UNSTABLE_API_MESSAGE + + +@pytest.fixture +def ds_envelope(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette([db_path]) + ds.root_enabled = True + yield ds + ds.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/.json", + "/-/.json", + "/-/versions.json", + "/-/settings.json", + "/-/config.json", + "/-/actor.json", + "/-/jump.json", + "/-/schema.json", + "/fixtures/-/schema.json", + "/fixtures/facetable/-/schema.json", + "/-/allowed.json?action=view-instance", + "/fixtures/facet_cities/-/autocomplete?_initial=1", + ), +) +async def test_success_object_has_ok_true(ds_client, path): + response = await ds_client.get(path) + assert response.status_code == 200 + data = response.json() + assert isinstance(data, dict) + assert data["ok"] is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/-/rules.json?action=view-instance", + "/-/check.json?action=view-instance", + "/-/threads.json", + ), +) +async def test_permission_debug_success_has_ok_true(ds_envelope, path): + response = await ds_envelope.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + + +@pytest.mark.asyncio +async def test_permissions_post_success_has_ok_true(ds_envelope): + response = await ds_envelope.client.post( + "/-/permissions", + data={"actor": '{"id": "root"}', "permission": "view-instance"}, + actor={"id": "root"}, + ) + assert response.status_code == 200 + assert response.json()["ok"] is True + + +@pytest.mark.asyncio +async def test_plugins_json_is_array(ds_client): + response = await ds_client.get("/-/plugins.json") + assert response.status_code == 200 + data = response.json() + assert isinstance(data, list) + assert all(isinstance(plugin, dict) for plugin in data) + # ?all=1 should include Datasette's default plugins in the same shape + response_all = await ds_client.get("/-/plugins.json?all=1") + all_plugins = response_all.json() + assert len(all_plugins) > len(data) + + +@pytest.mark.asyncio +async def test_databases_json_is_object(ds_client): + response = await ds_client.get("/-/databases.json") + assert response.status_code == 200 + data = response.json() + assert set(data.keys()) == {"ok", "databases"} + assert data["ok"] is True + assert isinstance(data["databases"], list) + assert "fixtures" in {db["name"] for db in data["databases"]} + + +@pytest.mark.asyncio +async def test_actions_json_is_object(ds_envelope): + response = await ds_envelope.client.get("/-/actions.json", actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert set(data.keys()) == {"ok", "actions"} + assert data["ok"] is True + assert isinstance(data["actions"], list) + assert "view-instance" in {action["name"] for action in data["actions"]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ( + "/.json", + "/-/.json", + "/fixtures/-/queries/analyze?sql=select+1", + "/fixtures/-/query/parameters?sql=select+:name", + "/fixtures/-/execute-write/analyze?sql=delete+from+facetable", + ), +) +async def test_undocumented_endpoints_report_unstable(ds_client, path): + ds_client.ds.root_enabled = True + try: + response = await ds_client.get(path, actor={"id": "root"}) + finally: + ds_client.ds.root_enabled = False + assert response.status_code == 200 + assert response.json()["unstable"] == UNSTABLE_API_MESSAGE + + +@pytest.mark.asyncio +async def test_query_store_and_definition_report_unstable(ds_envelope): + store = await ds_envelope.client.post( + "/data/-/queries/store", + json={"query": {"name": "unstable_check", "sql": "select 1"}}, + actor={"id": "root"}, + ) + assert store.status_code == 201 + assert store.json()["unstable"] == UNSTABLE_API_MESSAGE + definition = await ds_envelope.client.get( + "/data/unstable_check/-/definition", actor={"id": "root"} + ) + assert definition.status_code == 200 + assert definition.json()["unstable"] == UNSTABLE_API_MESSAGE + + +@pytest.mark.asyncio +async def test_permissions_post_reports_unstable(ds_envelope): + response = await ds_envelope.client.post( + "/-/permissions", + data={"actor": '{"id": "root"}', "permission": "view-instance"}, + actor={"id": "root"}, + ) + assert response.status_code == 200 + assert response.json()["unstable"] == UNSTABLE_API_MESSAGE + + +@pytest.mark.asyncio +async def test_documented_endpoints_do_not_report_unstable(ds_client): + for path in ("/-/versions.json", "/fixtures.json", "/fixtures/facetable.json"): + response = await ds_client.get(path) + assert response.status_code == 200 + assert "unstable" not in response.json() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 272e39e3..b8c6be87 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -31,8 +31,8 @@ async def test_table_not_exists_json(ds_client): assert (await ds_client.get("/fixtures/blah.json")).json() == { "ok": False, "error": "Table not found", + "errors": ["Table not found"], "status": 404, - "title": None, } @@ -123,8 +123,8 @@ async def test_html_only_extras_are_not_available_via_json(ds_client, extra): # These extras exist for the HTML view; their values are not JSON # serializable so they are internal, not part of the JSON API response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}") - assert response.status_code == 200 - assert extra not in response.json() + assert response.status_code == 400 + assert response.json()["errors"] == [f"Unknown _extra: {extra}"] @pytest.mark.asyncio @@ -180,9 +180,11 @@ def test_query_extra_query_reports_bound_params(): assert response.json["query"]["params"] == {} -def test_query_extra_query_does_not_echo_querystring_without_sql(): +def test_query_extra_query_does_not_echo_querystring(): with make_app_client() as client: - response = client.get("/fixtures/-/query.json?_extra=query&foo=bar") + response = client.get( + "/fixtures/-/query.json?sql=select+1&_extra=query&foo=bar" + ) assert response.status == 200 assert response.json["query"]["params"] == {} @@ -242,8 +244,8 @@ async def test_table_shape_invalid(ds_client): assert response.json() == { "ok": False, "error": "Invalid _shape: invalid", + "errors": ["Invalid _shape: invalid"], "status": 400, - "title": None, } @@ -321,10 +323,6 @@ async def test_paginate_tables_and_views( fetched = [] count = 0 while path: - if "?" in path: - path += "&_extra=next_url" - else: - path += "?_extra=next_url" response = await ds_client.get(path) assert response.status_code == 200 count += 1 @@ -357,9 +355,7 @@ async def test_validate_page_size(ds_client, path, expected_error): @pytest.mark.asyncio async def test_page_size_zero(ds_client): """For _size=0 we return the counts, empty rows and no continuation token""" - response = await ds_client.get( - "/fixtures/no_primary_key.json?_size=0&_extra=count,next_url" - ) + response = await ds_client.get("/fixtures/no_primary_key.json?_size=0&_extra=count") assert response.status_code == 200 assert [] == response.json()["rows"] assert 202 == response.json()["count"] @@ -370,7 +366,7 @@ async def test_page_size_zero(ds_client): @pytest.mark.asyncio async def test_paginate_compound_keys(ds_client): fetched = [] - path = "/fixtures/compound_three_primary_keys.json?_shape=objects&_extra=next_url" + path = "/fixtures/compound_three_primary_keys.json?_shape=objects" page = 0 while path: page += 1 @@ -391,7 +387,9 @@ async def test_paginate_compound_keys(ds_client): @pytest.mark.asyncio async def test_paginate_compound_keys_with_extra_filters(ds_client): fetched = [] - path = "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects&_extra=next_url" + path = ( + "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects" + ) page = 0 while path: page += 1 @@ -444,7 +442,7 @@ async def test_paginate_compound_keys_with_extra_filters(ds_client): ], ) async def test_sortable(ds_client, query_string, sort_key, human_description_en): - path = f"/fixtures/sortable.json?_shape=objects&_extra=human_description_en,next_url&{query_string}" + path = f"/fixtures/sortable.json?_shape=objects&_extra=human_description_en&{query_string}" fetched = [] page = 0 while path: @@ -635,8 +633,8 @@ async def test_searchable_invalid_column(ds_client): assert response.json() == { "ok": False, "error": "Cannot search by that column", + "errors": ["Cannot search by that column"], "status": 400, - "title": None, } @@ -775,7 +773,7 @@ async def test_table_filter_extra_where_invalid(ds_client): "/fixtures/facetable.json?_where=_neighborhood=Dogpatch'" ) assert response.status_code == 400 - assert "Invalid SQL" == response.json()["title"] + assert "unrecognized token" in response.json()["error"] def test_table_filter_extra_where_disabled_if_no_sql_allowed(): @@ -848,7 +846,7 @@ def test_page_size_matching_max_returned_rows( app_client_returned_rows_matches_page_size, ): fetched = [] - path = "/fixtures/no_primary_key.json?_extra=next_url" + path = "/fixtures/no_primary_key.json" while path: response = app_client_returned_rows_matches_page_size.get(path) fetched.extend(response.json["rows"]) @@ -1336,6 +1334,109 @@ async def test_binary_data_in_json(ds_client, path, expected_json, expected_text assert response.text == expected_text +@pytest.mark.asyncio +async def test_column_details_extra_table(ds_client): + response = await ds_client.get( + "/fixtures/binary_data.json?_size=0&_extra=column_details" + ) + assert response.status_code == 200 + data_detail = response.json()["column_details"]["data"] + assert data_detail["type"].lower() == "blob" + assert data_detail == { + "type": data_detail["type"], + "sqlite_type": "BLOB", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, + } + + response = await ds_client.get( + "/fixtures/simple_primary_key.json?_size=0&_extra=column_details" + ) + assert response.status_code == 200 + column_details = response.json()["column_details"] + id_detail = column_details["id"] + assert id_detail["type"].lower() == "integer" + assert id_detail == { + "type": id_detail["type"], + "sqlite_type": "INTEGER", + "notnull": False, + "default": None, + "is_pk": True, + "pk_position": 1, + "hidden": 0, + } + content_detail = column_details["content"] + assert content_detail["type"].lower() == "text" + assert content_detail == { + "type": content_detail["type"], + "sqlite_type": "TEXT", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, + } + + response = await ds_client.get( + "/fixtures/compound_three_primary_keys.json?_size=0&_extra=column_details" + ) + assert response.status_code == 200 + column_details = response.json()["column_details"] + assert column_details["pk1"]["is_pk"] is True + assert column_details["pk1"]["pk_position"] == 1 + assert column_details["pk2"]["is_pk"] is True + assert column_details["pk2"]["pk_position"] == 2 + assert column_details["pk3"]["is_pk"] is True + assert column_details["pk3"]["pk_position"] == 3 + assert column_details["content"]["is_pk"] is False + assert column_details["content"]["pk_position"] == 0 + + +def test_column_details_extra_defaults_and_notnull(): + with make_app_client(extra_databases={"defaults.db": """ + CREATE TABLE defaults ( + i INTEGER NOT NULL DEFAULT 42, + s TEXT DEFAULT 'hello', + dt TEXT DEFAULT (datetime('now')) + ); + """}) as client: + response = client.get("/defaults/defaults.json?_size=0&_extra=column_details") + assert response.status == 200 + column_details = response.json["column_details"] + assert column_details["i"]["notnull"] is True + assert column_details["i"]["default"] == "42" + assert column_details["s"]["notnull"] is False + assert column_details["s"]["default"] == "'hello'" + assert column_details["dt"]["default"] == "datetime('now')" + + +@pytest.mark.skipif( + sqlite_version() < (3, 31, 0), + reason="generated columns were added in SQLite 3.31.0", +) +def test_column_details_extra_generated_columns(): + with make_app_client(extra_databases={"generated.db": """ + CREATE TABLE generated_columns ( + body TEXT, + body_length_virtual INTEGER + GENERATED ALWAYS AS (length(body)) VIRTUAL, + body_length_stored INTEGER + GENERATED ALWAYS AS (length(body)) STORED + ); + """}) as client: + response = client.get( + "/generated/generated_columns.json?_size=0&_extra=column_details" + ) + assert response.status == 200 + column_details = response.json["column_details"] + assert column_details["body"]["hidden"] == 0 + assert column_details["body_length_virtual"]["hidden"] == 2 + assert column_details["body_length_stored"]["hidden"] == 3 + + @pytest.mark.asyncio @pytest.mark.parametrize( "qs", @@ -1489,6 +1590,7 @@ async def test_col_nocol_errors(ds_client, path, expected_error): { "ok": True, "next": None, + "next_url": None, "columns": ["id", "content", "content2"], "rows": [{"id": "1", "content": "hey", "content2": "world"}], "truncated": False, @@ -1499,9 +1601,11 @@ async def test_col_nocol_errors(ds_client, path, expected_error): { "ok": True, "next": None, + "next_url": None, "rows": [{"id": "1", "content": "hey", "content2": "world"}], "truncated": False, "count": 1, + "count_truncated": False, }, ), ), @@ -1588,3 +1692,58 @@ async def test_extra_render_cell(): finally: ds.pm.unregister(name="TestRenderCellPlugin") + + +@pytest.mark.asyncio +async def test_count_truncated_included_with_count_extra(tmp_path_factory): + from datasette.app import Datasette + from datasette.utils import sqlite3 + + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "counts.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table big (id integer primary key)") + conn.execute("create table small (id integer primary key)") + conn.executemany("insert into big (id) values (?)", [(i,) for i in range(10)]) + conn.executemany("insert into small (id) values (?)", [(i,) for i in range(3)]) + conn.commit() + conn.close() + ds = Datasette([db_path]) + ds.get_database("counts").count_limit = 5 + try: + response = await ds.client.get("/counts/big.json?_extra=count") + data = response.json() + # Count is capped at count_limit + 1 and flagged as truncated + assert data["count"] == 6 + assert data["count_truncated"] is True + + response = await ds.client.get("/counts/small.json?_extra=count") + data = response.json() + assert data["count"] == 3 + assert data["count_truncated"] is False + + # count_truncated can also be requested on its own + response = await ds.client.get("/counts/big.json?_extra=count_truncated") + assert response.json()["count_truncated"] is True + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_next_url_included_by_default(ds_client): + response = await ds_client.get("/fixtures/compound_three_primary_keys.json") + data = response.json() + assert data["next"] is not None + assert data["next_url"].endswith( + "/fixtures/compound_three_primary_keys.json?_next=" + + urllib.parse.quote(data["next"], safe="") + ) + # Follow to the last page - next and next_url are both null there + while data["next"]: + response = await ds_client.get( + "/fixtures/compound_three_primary_keys.json?_next=" + data["next"] + ) + data = response.json() + assert data["next"] is None + assert data["next_url"] is None diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 3af2bb08..2cfdd9f0 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1027,6 +1027,7 @@ async def test_database_create_table_action_button_and_data(): "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], "defaultExpressions": DEFAULT_EXPRESSION_OPTIONS, + "canInsertRows": False, }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] @@ -1050,6 +1051,40 @@ async def test_database_create_table_action_button_and_data(): ds.close() +@pytest.mark.asyncio +async def test_database_create_table_data_includes_insert_row_permission(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + "insert-row": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_insert_permission"), + name="data", + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[ + "createTable" + ] + assert create_table_data["canInsertRows"] is True + finally: + ds.close() + + @pytest.mark.asyncio async def test_database_create_table_data_includes_custom_column_types(): ds = Datasette( @@ -1316,6 +1351,7 @@ async def test_table_insert_action_button_and_data(): assert insert_data["path"] == "/data/items/-/insert" assert insert_data["tableName"] == "items" assert insert_data["primaryKeys"] == ["id"] + assert insert_data["maxInsertRows"] == 100 assert [column["name"] for column in insert_data["columns"]] == [ "name", "score", @@ -1629,7 +1665,7 @@ async def test_row_update_sets_message(): json={"update": {"name": long_name}, "return": True}, ) assert response.status_code == 200 - assert response.json()["row"]["name"] == long_name + assert response.json()["rows"][0]["name"] == long_name assert ds.unsign(response.cookies["ds_messages"], "messages") == [ ["Updated row 1 ({})".format(truncated_name), ds.INFO] ] @@ -1979,8 +2015,8 @@ async def test_sort_errors(ds_client, json, params, error): assert response.json() == { "ok": False, "error": error, + "errors": [error], "status": 400, - "title": None, } else: assert error in response.text @@ -2313,6 +2349,7 @@ async def test_foreign_key_labels_obey_permissions(config): assert root_b.json() == { "ok": True, "next": None, + "next_url": None, "rows": [{"id": 1, "name": "world", "a_id": {"value": 1, "label": "hello"}}], "truncated": False, } @@ -2320,6 +2357,7 @@ async def test_foreign_key_labels_obey_permissions(config): assert anon_b.json() == { "ok": True, "next": None, + "next_url": None, "rows": [{"id": 1, "name": "world", "a_id": 1}], "truncated": False, } diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 5c87f577..f5bbfead 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -5,7 +5,12 @@ Tests for the register_token_handler plugin hook. from datasette.app import Datasette from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.tokens import TokenHandler, TokenRestrictions, SignedTokenHandler +from datasette.tokens import ( + TokenHandler, + TokenInvalid, + TokenRestrictions, + SignedTokenHandler, +) import pytest @@ -66,10 +71,10 @@ async def test_verify_token_unknown_returns_none(datasette): @pytest.mark.asyncio -async def test_verify_token_bad_signature_returns_none(datasette): - """verify_token() should return None for tokens with bad signatures.""" - result = await datasette.verify_token("dstok_tampered_data_here") - assert result is None +async def test_verify_token_bad_signature_raises(datasette): + """verify_token() should raise TokenInvalid for tokens with bad signatures.""" + with pytest.raises(TokenInvalid): + await datasette.verify_token("dstok_tampered_data_here") @pytest.mark.asyncio @@ -334,5 +339,6 @@ async def test_signed_tokens_disabled(): ds = Datasette(settings={"allow_signed_tokens": False}) with pytest.raises(ValueError, match="Signed tokens are not enabled"): await ds.create_token("test_actor", handler="signed") - # verify_token should return None rather than raising - assert await ds.verify_token("dstok_anything") is None + # verify_token should raise TokenInvalid for a dstok_ token + with pytest.raises(TokenInvalid, match="not enabled"): + await ds.verify_token("dstok_anything") diff --git a/tests/test_utils.py b/tests/test_utils.py index 38ebb51e..a535ca93 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -132,7 +132,11 @@ def test_path_from_row_pks(row, pks, expected_path): """ {"CategoryID": 1, "Description": "Soft drinks", "Picture": {"$base64": true, "encoded": "FRwCx60F/g=="}} """.strip(), - ) + ), + ( + {"message": b"hello"}, + '{"message": {"$base64": true, "encoded": "aGVsbG8="}}', + ), ], ) def test_custom_json_encoder(obj, expected): @@ -752,6 +756,21 @@ def test_parse_metadata(content, expected): ("select 1 + :one + :two", ["one", "two"]), ("select 'bob' || '0:00' || :cat", ["cat"]), ("select this is invalid :one, :two, :three", ["one", "two", "three"]), + # A string literal containing a comment marker should not hide + # parameters that come after it + ("select * from t where note = '-- TODO' and id = :id", ["id"]), + ("select '--' || :y", ["y"]), + ("select * from t where note = '/* x */' and id = :id", ["id"]), + # Parameters that live inside a comment should be ignored + ("select :x -- and :ignored", ["x"]), + ("select :x /* and :ignored */ from t", ["x"]), + ("select :x /* and :ignored", ["x"]), + # Parameters inside quoted identifiers should be ignored + ("select [a:b] from t where id = :id", ["id"]), + ("select `a:b` from t where id = :id", ["id"]), + ("select `a``:b` from t where id = :id", ["id"]), + # Parameters inside a string literal should be ignored + ("select ':ignored' || :real", ["real"]), ), ) @pytest.mark.parametrize("use_async_version", (False, True))