diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 00000000..e56d9c27 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b0640ae8..7349a1ab 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v6 with: @@ -57,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable stored query demo + - name: And the counters writable canned query demo run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index b8fb8aaa..a54bd83a 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request: + pull_request_target: types: - opened diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 735e14e9..77cce7d1 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v6 - - uses: actions/cache@v5 + uses: actions/checkout@v4 + - uses: actions/cache@v4 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87300593..2e8cea9c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: @@ -56,7 +56,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index e622ef4c..afe8d6b2 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 9a808194..d42ae96b 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 59b5fbc0..3119d617 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v5 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index c514048e..1b3d2f2c 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 5162c47a..b490a9bf 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +20,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v5 + uses: actions/cache@v4 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index 23fce459..c81a3c0b 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,7 +25,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1b2e9d2..a0f5477b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index a033cd92..fcee0f21 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 72af1eec..123f6c71 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py deleted file mode 100644 index 5fb6b473..00000000 --- a/datasette/_pytest_plugin.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Pytest plugin that automatically closes any Datasette instances constructed -during a pytest test — both in the test body and in function-scoped -fixtures. Instances constructed by session-, module-, class- or package- -scoped fixtures are left alone, because other tests in the session will -still want to use them. - -Registered as a pytest11 entry point in pyproject.toml so that downstream -projects using Datasette get the same FD-safety net for their own tests. - -Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the -equivalent ini file). -""" - -from __future__ import annotations - -import contextvars -import weakref - -import pytest - -from datasette.app import Datasette - -_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( - "datasette_active_instances", default=None -) - -_original_init = Datasette.__init__ - - -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) - - -Datasette.__init__ = _tracking_init - - -def pytest_addoption(parser): - parser.addini( - "datasette_autoclose", - help=( - "Automatically close Datasette instances created inside test " - "bodies and function-scoped fixtures (default: true)." - ), - default="true", - ) - - -def _enabled(config) -> bool: - value = config.getini("datasette_autoclose") - if isinstance(value, bool): - return value - return str(value).strip().lower() not in ("false", "0", "no", "off") - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - """Track Datasette instances across setup, call and teardown; close at end.""" - if not _enabled(item.config): - yield - return - refs: list[weakref.ref] = [] - token = _active_instances.set(refs) - try: - yield - finally: - _active_instances.reset(token) - for ref in reversed(refs): - ds = ref() - if ds is None: - continue - try: - ds.close() - except Exception as e: - item.warn( - pytest.PytestUnraisableExceptionWarning( - f"Error closing Datasette instance: {e!r}" - ) - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef, request): - """Exempt instances created by non-function-scoped fixtures. - - Session-, module-, class- and package-scoped fixtures produce Datasette - instances that must survive beyond the current test — other tests in - the session will still use them. When such a fixture creates one or - more Datasette instances during its setup, we snapshot the tracking - list before the fixture runs and subtract off any instances that were - added during its setup, so they don't get closed at test teardown. - """ - refs = _active_instances.get() - if refs is None: - yield - return - before_ids = {id(ref) for ref in refs} - yield - if fixturedef.scope != "function": - new_refs = [ref for ref in refs if id(ref) not in before_ids] - for new_ref in new_refs: - try: - refs.remove(new_ref) - except ValueError: - pass diff --git a/datasette/app.py b/datasette/app.py index 56b89789..4c98e521 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,5 +1,6 @@ from __future__ import annotations +from asgi_csrf import Errors import asyncio import contextvars from typing import TYPE_CHECKING, Any, Dict, Iterable, List @@ -7,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, List if TYPE_CHECKING: from datasette.permissions import Resource from datasette.tokens import TokenRestrictions +import asgi_csrf import collections import dataclasses import datetime @@ -42,25 +44,8 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType -from . import stored_queries from .views import Context -from .views.database import ( - database_download, - DatabaseView, - TableCreateView, - QueryView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) +from .views.database import database_download, DatabaseView, TableCreateView, QueryView from .views.index import IndexView from .views.special import ( JsonDataView, @@ -75,7 +60,7 @@ from .views.special import ( AllowedResourcesView, PermissionRulesView, PermissionCheckView, - JumpView, + TablesView, InstanceSchemaView, DatabaseSchemaView, TableSchemaView, @@ -135,7 +120,6 @@ from .utils.asgi import ( asgi_send_file, asgi_send_redirect, ) -from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -343,7 +327,6 @@ class Datasette: default_deny=False, ): self._startup_invoked = False - self._closed = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -398,7 +381,7 @@ class Datasette: self.internal_db_created = False if internal is None: - self._internal_database = Database(self, is_temp_disk=True) + self._internal_database = Database(self, memory_name=secrets.token_hex()) else: self._internal_database = Database(self, path=internal, mode="rwc") self._internal_database.name = INTERNAL_DB_NAME @@ -588,9 +571,6 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file - async def _save_queries_from_config(self): - await stored_queries.save_queries_from_config(self) - def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: @@ -634,36 +614,15 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } - catalog_table_names = ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - "catalog_databases", - ) # Delete stale entries for databases that are no longer attached - catalog_database_names = set(current_schema_versions.keys()) - for table in catalog_table_names[:-1]: - catalog_database_names.update( - row["database_name"] - for row in await internal_db.execute( - "select distinct database_name from {}".format(table) - ) - if row["database_name"] is not None + stale_databases = set(current_schema_versions.keys()) - set( + self.databases.keys() + ) + for stale_db_name in stale_databases: + await internal_db.execute_write( + "DELETE FROM catalog_databases WHERE database_name = ?", + [stale_db_name], ) - stale_databases = catalog_database_names - set(self.databases.keys()) - if stale_databases: - - def delete_stale_database_catalog(conn): - for stale_db_name in stale_databases: - for table in catalog_table_names: - conn.execute( - "DELETE FROM {} WHERE database_name = ?".format(table), - [stale_db_name], - ) - - await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it @@ -751,7 +710,6 @@ class Datasette: await await_me_maybe(hook) # Ensure internal tables and metadata are populated before startup hooks await self._refresh_schemas() - await self._save_queries_from_config() # Load column_types from config into internal DB await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): @@ -877,33 +835,6 @@ class Datasette: new_databases.pop(name) self.databases = new_databases - def close(self): - """Release all resources held by this Datasette instance. - - Closes every attached Database (including the internal database), - shuts down the executor, and unlinks the temporary file used for - the internal database if one was created. Idempotent and one-way. - """ - if self._closed: - return - self._closed = True - first_exception = None - dbs = list(self.databases.values()) + [self._internal_database] - for db in dbs: - try: - db.close() - except Exception as e: - if first_exception is None: - first_exception = e - if self.executor is not None: - try: - self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: - if first_exception is None: - first_exception = e - if first_exception is not None: - raise first_exception - def setting(self, key): return self._settings.get(key, None) @@ -1028,179 +959,6 @@ class Datasette: [database_name, resource_name, column_name, key, value], ) - @staticmethod - def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: - return stored_queries.query_row_to_stored_query(row) - - @staticmethod - def _query_options_json(options): - return stored_queries.query_options_json(options) - - async def add_query( - self, - database: str, - name: str, - sql: str, - *, - title: str | None = None, - description: str | None = None, - description_html: str | None = None, - hide_sql: bool = False, - fragment: str | None = None, - parameters: Iterable[str] | None = None, - is_write: bool = False, - is_private: bool = False, - is_trusted: bool = False, - source: str = "plugin", - owner_id: str | None = None, - on_success_message: str | None = None, - on_success_message_sql: str | None = None, - on_success_redirect: str | None = None, - on_error_message: str | None = None, - on_error_redirect: str | None = None, - replace: bool = True, - ) -> None: - return await stored_queries.add_query( - self, - database, - name, - sql, - title=title, - description=description, - description_html=description_html, - hide_sql=hide_sql, - fragment=fragment, - parameters=parameters, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - on_success_message=on_success_message, - on_success_message_sql=on_success_message_sql, - on_success_redirect=on_success_redirect, - on_error_message=on_error_message, - on_error_redirect=on_error_redirect, - replace=replace, - ) - - async def update_query( - self, - database: str, - name: str, - *, - sql=stored_queries.UNCHANGED, - title=stored_queries.UNCHANGED, - description=stored_queries.UNCHANGED, - description_html=stored_queries.UNCHANGED, - hide_sql=stored_queries.UNCHANGED, - fragment=stored_queries.UNCHANGED, - parameters=stored_queries.UNCHANGED, - is_write=stored_queries.UNCHANGED, - is_private=stored_queries.UNCHANGED, - is_trusted=stored_queries.UNCHANGED, - source=stored_queries.UNCHANGED, - owner_id=stored_queries.UNCHANGED, - on_success_message=stored_queries.UNCHANGED, - on_success_message_sql=stored_queries.UNCHANGED, - on_success_redirect=stored_queries.UNCHANGED, - on_error_message=stored_queries.UNCHANGED, - on_error_redirect=stored_queries.UNCHANGED, - ) -> None: - return await stored_queries.update_query( - self, - database, - name, - sql=sql, - title=title, - description=description, - description_html=description_html, - hide_sql=hide_sql, - fragment=fragment, - parameters=parameters, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - on_success_message=on_success_message, - on_success_message_sql=on_success_message_sql, - on_success_redirect=on_success_redirect, - on_error_message=on_error_message, - on_error_redirect=on_error_redirect, - ) - - async def remove_query( - self, database: str, name: str, source: str | None = None - ) -> None: - return await stored_queries.remove_query(self, database, name, source=source) - - async def get_query( - self, database: str, name: str - ) -> stored_queries.StoredQuery | None: - return await stored_queries.get_query(self, database, name) - - async def count_queries( - self, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - ) -> int: - return await stored_queries.count_queries( - self, - database, - actor=actor, - q=q, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - ) - - async def list_queries( - self, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - limit: int = 50, - cursor: str | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - include_private: bool = False, - ) -> stored_queries.StoredQueryPage: - return await stored_queries.list_queries( - self, - database, - actor=actor, - limit=limit, - cursor=cursor, - q=q, - is_write=is_write, - is_private=is_private, - is_trusted=is_trusted, - source=source, - owner_id=owner_id, - include_private=include_private, - ) - - async def ensure_query_write_permissions( - self, database, sql, *, actor=None, params=None, analysis=None - ): - return await stored_queries.ensure_query_write_permissions( - self, database, sql, actor=actor, params=params, analysis=analysis - ) - # Column types API async def _get_resource_column_details(self, database: str, resource: str): @@ -1413,24 +1171,36 @@ class Datasette: return db_plugin_config - def static_hash(self, filename): - if not hasattr(self, "_static_hashes"): - self._static_hashes = {} - path = os.path.join(str(app_root), "datasette/static", filename) - signature = (os.path.getmtime(path), os.path.getsize(path)) - cached = self._static_hashes.get(filename) - if cached and cached["signature"] == signature: - return cached["hash"] - with open(path) as fp: - static_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[:6] - self._static_hashes[filename] = { - "signature": signature, - "hash": static_hash, - } - return static_hash - def app_css_hash(self): - return self.static_hash("app.css") + if not hasattr(self, "_app_css_hash"): + with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: + self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ + :6 + ] + return self._app_css_hash + + async def get_canned_queries(self, database_name, actor): + queries = {} + for more_queries in pm.hook.canned_queries( + datasette=self, + database=database_name, + actor=actor, + ): + more_queries = await await_me_maybe(more_queries) + queries.update(more_queries or {}) + # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}} + for key in queries: + if not isinstance(queries[key], dict): + queries[key] = {"sql": queries[key]} + # Also make sure "name" is available: + queries[key]["name"] = key + return queries + + async def get_canned_query(self, database_name, query_name, actor): + queries = await self.get_canned_queries(database_name, actor) + query = queries.get(query_name) + if query: + return query def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row @@ -2050,7 +1820,6 @@ class Datasette: break except importlib.metadata.PackageNotFoundError: pass - conn.close() return info def _plugins(self, request=None, all=False): @@ -2234,11 +2003,7 @@ class Datasette: "extra_js_urls", template, context, request, view_name ), "base_url": self.setting("base_url"), - "csrftoken": ( - request.scope["csrftoken"] - if request and "csrftoken" in request.scope - else lambda: "" - ), + "csrftoken": request.scope["csrftoken"] if request else lambda: "", "datasette_version": __version__, }, **extra_template_vars, @@ -2404,12 +2169,8 @@ class Datasette: r"/-/api$", ) add_route( - JumpView.as_view(self), - r"/-/jump(\.(?Pjson))?$", - ) - add_route( - GlobalQueryListView.as_view(self), - r"/-/queries(\.(?Pjson))?$", + TablesView.as_view(self), + r"/-/tables(\.(?Pjson))?$", ) add_route( InstanceSchemaView.as_view(self), @@ -2456,50 +2217,14 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") - add_route( - QueryListView.as_view(self), - r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", - ) - add_route( - QueryCreateAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/analyze$", - ) - add_route( - QueryStoreView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/store$", - ) - add_route( - ExecuteWriteAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write/analyze$", - ) - add_route( - ExecuteWriteView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write$", - ) add_route( DatabaseSchemaView.as_view(self), r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", ) - add_route( - QueryParametersView.as_view(self), - r"/(?P[^\/\.]+)/-/query/parameters$", - ) add_route( wrap_view(QueryView, self), r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", ) - add_route( - QueryDefinitionView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", - ) - add_route( - QueryUpdateView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", - ) - add_route( - QueryDeleteView.as_view(self), - r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/delete$", - ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", @@ -2581,13 +2306,29 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) - async def _close_on_shutdown(): - self.close() + async def custom_csrf_error(scope, send, message_id): + await asgi_send( + send, + content=await self.render_template( + "csrf_error.html", + {"message_id": message_id, "message_name": Errors(message_id).name}, + ), + status=403, + content_type="text/html; charset=utf-8", + ) - asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) + asgi = asgi_csrf.asgi_csrf( + DatasetteRouter(self, routes), + signing_secret=self._secret, + cookie_name="ds_csrftoken", + skip_if_scope=lambda scope: any( + pm.hook.skip_csrf(datasette=self, scope=scope) + ), + send_csrf_failed=custom_csrf_error, + ) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiLifespan(asgi) asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) @@ -2929,21 +2670,9 @@ class DatasetteClient: path = f"http://localhost{path}" return path - def _apply_actor(self, kwargs): - """If ``actor=`` was supplied, convert it into a signed ds_actor cookie.""" - actor = kwargs.pop("actor", None) - if actor is None: - return - cookies = dict(kwargs.get("cookies") or {}) - if "ds_actor" in cookies: - raise TypeError("Cannot pass both actor= and a ds_actor cookie") - cookies["ds_actor"] = self.actor_cookie(actor) - kwargs["cookies"] = cookies - async def _request(self, method, path, skip_permission_checks=False, **kwargs): from datasette.permissions import SkipPermissions - self._apply_actor(kwargs) with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): @@ -3009,7 +2738,6 @@ class DatasetteClient: from datasette.permissions import SkipPermissions avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) - self._apply_actor(kwargs) with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): diff --git a/datasette/cli.py b/datasette/cli.py index 93aa22ef..32a4d898 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -615,9 +615,7 @@ def serve( for file in file_paths: if not pathlib.Path(file).exists(): if create: - conn = sqlite3.connect(file) - conn.execute("vacuum") - conn.close() + sqlite3.connect(file).execute("vacuum") else: raise click.ClickException( "Invalid value for '[FILES]...': Path '{}' does not exist.".format( diff --git a/datasette/csrf.py b/datasette/csrf.py deleted file mode 100644 index df239aee..00000000 --- a/datasette/csrf.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Header-based CSRF (Cross-Origin) protection. - -Datasette uses the Sec-Fetch-Site + Origin header approach described in -Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented -in Go 1.25's http.CrossOriginProtection. This replaces the previous -token-based asgi-csrf mechanism. -""" - -from __future__ import annotations - -import secrets -import urllib.parse - -from .utils.asgi import asgi_send - -SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) - -DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443} - - -def _normalize_headers(raw_headers): - """Lowercase header names; for duplicates, last value wins.""" - result = {} - for name, value in raw_headers: - if isinstance(name, str): - name = name.encode("latin-1") - if isinstance(value, str): - value = value.encode("latin-1") - result[name.lower()] = value - return result - - -def _origin_tuple(value): - """ - Parse an origin-like string into ``(scheme, host, port)`` with default - ports filled in. Raises ``ValueError`` for malformed input. - """ - parsed = urllib.parse.urlsplit(value) - scheme = (parsed.scheme or "").lower() - host = (parsed.hostname or "").lower() - if not scheme or not host: - raise ValueError("missing scheme or host in {!r}".format(value)) - port = parsed.port # may raise ValueError on bad ports - if port is None: - port = DEFAULT_PORTS.get(scheme) - if port is None: - raise ValueError("unknown default port for scheme {!r}".format(scheme)) - return scheme, host, port - - -def _install_legacy_csrftoken(scope): - """ - Populate ``scope["csrftoken"]`` with a callable returning a per-request - random token. Provided for plugin compatibility only - core no longer - uses this value for CSRF enforcement. - """ - - def csrftoken(): - if "_datasette_legacy_csrftoken" not in scope: - scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32) - return scope["_datasette_legacy_csrftoken"] - - scope["csrftoken"] = csrftoken - - -class CrossOriginProtectionMiddleware: - """ - Modern CSRF protection using the Sec-Fetch-Site and Origin headers. - - Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's - http.CrossOriginProtection. See https://words.filippo.io/csrf/ - - Unsafe-method requests are allowed through only if they look same-origin. - Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin - and are passed through unchanged - CSRF is a browser-only attack. - """ - - SAFE_METHODS = SAFE_METHODS - - def __init__(self, app, datasette): - self.app = app - self.datasette = datasette - - async def __call__(self, scope, receive, send): - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - _install_legacy_csrftoken(scope) - - if scope.get("method", "GET") in self.SAFE_METHODS: - await self.app(scope, receive, send) - return - - headers = _normalize_headers(scope.get("headers") or []) - - authorization = headers.get(b"authorization", b"").decode("latin-1") - cookie_header = headers.get(b"cookie") - # Bearer-token requests are not ambient browser credentials, so they - # are not CSRF-vulnerable. Narrowly exempt them from the header check - # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; - # schemes like Basic or Digest can be browser-managed and ambient. - # If the request also carries a Cookie header, ambient cookie auth - # could be in play, so do NOT treat it as exempt. - if authorization and not cookie_header: - parts = authorization.split(None, 1) - if parts and parts[0].lower() == "bearer": - await self.app(scope, receive, send) - return - - origin_bytes = headers.get(b"origin") - sec_fetch_site_bytes = headers.get(b"sec-fetch-site") - host_bytes = headers.get(b"host", b"") - origin = origin_bytes.decode("latin-1") if origin_bytes else None - sec_fetch_site = ( - sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None - ) - host = host_bytes.decode("latin-1") - - # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS) - if sec_fetch_site is not None: - if sec_fetch_site in ("same-origin", "none"): - await self.app(scope, receive, send) - return - await self._forbid( - send, - "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( - sec_fetch_site - ), - ) - return - - # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.) - if origin is None: - await self.app(scope, receive, send) - return - - # Fallback for older browsers: Origin must match the request's own - # scheme + host + port. Compare full origin tuples, not host alone. - request_scheme = self._request_scheme(scope) - try: - origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) - except ValueError: - await self._forbid( - send, - "Malformed Origin {!r} or Host {!r}".format(origin, host), - ) - return - - if origin_tuple == expected_tuple: - await self.app(scope, receive, send) - return - - await self._forbid( - send, - "Origin {!r} does not match Host {!r}".format(origin, host), - ) - - def _request_scheme(self, scope): - if self.datasette is not None: - try: - if self.datasette.setting("force_https_urls"): - return "https" - except Exception: - pass - return scope.get("scheme") or "http" - - async def _forbid(self, send, reason): - await asgi_send( - send, - content=await self.datasette.render_template( - "csrf_error.html", {"reason": reason} - ), - status=403, - content_type="text/html; charset=utf-8", - ) diff --git a/datasette/database.py b/datasette/database.py index e7e9527e..fcf69c7f 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,19 +1,15 @@ import asyncio -import atexit from collections import namedtuple -import inspect -import os from pathlib import Path +import janus import queue import sqlite_utils import sys -import tempfile import threading import uuid from .tracer import trace from .utils import ( - call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, @@ -25,7 +21,6 @@ from .utils import ( table_columns, table_column_details, ) -from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables from .utils.sqlite import sqlite_version from .inspect import inspect_hash @@ -34,13 +29,6 @@ connections = threading.local() AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) -class DatasetteClosedError(RuntimeError): - """Raised when using a Datasette or Database instance after close().""" - - -_SHUTDOWN = object() - - class Database: # For table counts stop at this many rows: count_limit = 10000 @@ -54,7 +42,6 @@ class Database: is_memory=False, memory_name=None, mode=None, - is_temp_disk=False, ): self.name = None self._thread_local_id = f"x{self._thread_local_id_counter}" @@ -65,44 +52,19 @@ class Database: self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name - self.is_temp_disk = is_temp_disk if memory_name is not None: self.is_memory = True - if is_temp_disk: - fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_") - os.close(fd) - self.path = temp_path - self.is_mutable = True - self.mode = "rwc" - self._wal_enabled = False - atexit.register(self._cleanup_temp_file) - else: - self._wal_enabled = False self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None self._write_queue = None - self._closed = False - self._pending_execute_futures = set() - self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] - if not is_temp_disk: - self.mode = mode - - def _check_not_closed(self): - if self._closed: - raise DatasetteClosedError( - "Database {!r} has been closed".format(self.name) - ) - - def _remove_pending_execute_future(self, future): - with self._pending_execute_futures_lock: - self._pending_execute_futures.discard(future) + self.mode = mode @property def cached_table_counts(self): @@ -123,8 +85,6 @@ class Database: return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): - if self.is_temp_disk: - return "_temp_disk" if self.path: return Path(self.path).stem elif self.memory_name: @@ -163,82 +123,14 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) self._all_file_connections.append(conn) - if self.is_temp_disk and not self._wal_enabled: - conn.execute("PRAGMA journal_mode=WAL") - self._wal_enabled = True return conn def close(self): - """Release all resources held by this database. - - Idempotent. After close() further calls to execute()/execute_fn()/ - execute_write()/execute_write_fn() raise DatasetteClosedError. - """ - if self._closed: - return - with self._pending_execute_futures_lock: - if self._closed: - return - self._closed = True - pending_execute_futures = tuple(self._pending_execute_futures) - # Shut down the write thread, if any, via a sentinel. The thread - # drains any writes already queued before the sentinel and then - # closes its own write connection and returns. - write_thread = self._write_thread - if write_thread is not None and self._write_queue is not None: - self._write_queue.put(_SHUTDOWN) - write_thread.join(timeout=10) - if write_thread.is_alive(): - sys.stderr.write( - "Datasette: write thread for {!r} did not exit within 10s\n".format( - self.name - ) - ) - sys.stderr.flush() - for future in pending_execute_futures: - try: - future.result() - except Exception: - pass - # Close anything still tracked in _all_file_connections + # Close all connections - useful to avoid running out of file handles in tests for connection in self._all_file_connections: - try: - connection.close() - except Exception: - pass - self._all_file_connections = [] - # Drop per-thread cached read connections we can reach - try: - delattr(connections, self._thread_local_id) - except AttributeError: - pass - # Close non-threaded-mode cached connections if still open - if self._read_connection is not None: - try: - self._read_connection.close() - except Exception: - pass - self._read_connection = None - if self._write_connection is not None: - try: - self._write_connection.close() - except Exception: - pass - self._write_connection = None - if self.is_temp_disk: - self._cleanup_temp_file() - - def _cleanup_temp_file(self): - if self.is_temp_disk and self.path: - for suffix in ("", "-wal", "-shm"): - try: - os.unlink(self.path + suffix) - except OSError: - pass + connection.close() async def execute_write(self, sql, params=None, block=True, request=None): - self._check_not_closed() - def _inner(conn): return conn.execute(sql, params or []) @@ -247,8 +139,6 @@ class Database: return results async def execute_write_script(self, sql, block=True, request=None): - self._check_not_closed() - def _inner(conn): return conn.executescript(sql) @@ -259,8 +149,6 @@ class Database: return results async def execute_write_many(self, sql, params_seq, block=True, request=None): - self._check_not_closed() - def _inner(conn): count = 0 @@ -282,7 +170,6 @@ class Database: return results async def execute_isolated_fn(self, fn): - self._check_not_closed() # Open a new connection just for the duration of this function # blocking the write queue to avoid any writes occurring during it if self.ds.executor is None: @@ -302,21 +189,8 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) - async def analyze_sql(self, sql, params=None) -> SQLAnalysis: - self._check_not_closed() - - return await self.execute_isolated_fn( - lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) - ) - async def execute_write_fn(self, fn, block=True, transaction=True, request=None): - self._check_not_closed() - pending_events = [] - - def track_event(event): - pending_events.append(event) - - fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event) + fn = self._wrap_fn_with_hooks(fn, request, transaction) if self.ds.executor is None: # non-threaded mode if self._write_connection is None: @@ -324,53 +198,17 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: - result = fn(self._write_connection) + return fn(self._write_connection) else: - result = fn(self._write_connection) + return fn(self._write_connection) else: - result = await self._send_to_write_thread( + return await self._send_to_write_thread( fn, block=block, transaction=transaction ) - if block: - for event in pending_events: - await self.ds.track_event(event) - else: - # For non-blocking writes, spawn a background task to - # dispatch events after the write thread completes - task_id, reply_future = result - async def _dispatch_events_after_write(): - try: - await reply_future - except Exception: - # if the write failed, don't emit success events - return - for event in pending_events: - await self.ds.track_event(event) - - asyncio.ensure_future(_dispatch_events_after_write()) - result = task_id - return result - - def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): + def _wrap_fn_with_hooks(self, fn, request, transaction): from .plugins import pm - # Wrap fn so it receives track_event if its signature supports it. - # Historically fn was called positionally, so any single-parameter - # name (conn, connection, db, ...) worked. Preserve that by only - # switching to keyword dependency injection when the callback - # explicitly opts in by declaring a `track_event` parameter. - original_fn = fn - - if "track_event" in inspect.signature(original_fn).parameters: - - def fn_with_track_event(conn): - return call_with_supported_arguments( - original_fn, conn=conn, track_event=track_event - ) - - fn = fn_with_track_event - wrappers = pm.hook.write_wrapper( datasette=self.ds, database=self.name, @@ -382,9 +220,10 @@ class Database: return fn # Build the wrapped fn by nesting context manager generators. # The first wrapper returned by pluggy is outermost. + original_fn = fn for wrapper_factory in reversed(wrappers): - fn = _apply_write_wrapper(fn, wrapper_factory, track_event) - return fn + original_fn = _apply_write_wrapper(original_fn, wrapper_factory) + return original_fn async def _send_to_write_thread( self, fn, block=True, isolated_connection=False, transaction=True @@ -400,15 +239,18 @@ class Database: ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - loop = asyncio.get_running_loop() - reply_future = loop.create_future() + reply_queue = janus.Queue() self._write_queue.put( - WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) + WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) ) if block: - return await reply_future + result = await reply_queue.async_q.get() + if isinstance(result, Exception): + raise result + else: + return result else: - return task_id, reply_future + return task_id def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -422,47 +264,38 @@ class Database: conn_exception = e while True: task = self._write_queue.get() - if task is _SHUTDOWN: - if conn is not None: - try: - conn.close() - except Exception: - pass - return - exception = None - result = None if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass + result = conn_exception else: - try: - if task.transaction: - with conn: + if task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + else: + try: + if task.transaction: + with conn: + result = task.fn(conn) + else: result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): - self._check_not_closed() if self.ds.executor is None: # non-threaded mode if self._read_connection is None: @@ -479,12 +312,9 @@ class Database: setattr(connections, self._thread_local_id, conn) return fn(conn) - with self._pending_execute_futures_lock: - self._check_not_closed() - future = self.ds.executor.submit(in_thread) - self._pending_execute_futures.add(future) - future.add_done_callback(self._remove_pending_execute_future) - return await asyncio.wrap_future(future) + return await asyncio.get_event_loop().run_in_executor( + self.ds.executor, in_thread + ) async def execute( self, @@ -496,7 +326,6 @@ class Database: log_sql_errors=True, ): """Executes sql against db_name in a thread""" - self._check_not_closed() page_size = page_size or self.ds.page_size def sql_operation_in_thread(conn): @@ -544,7 +373,7 @@ class Database: def hash(self): if self.cached_hash is not None: return self.cached_hash - elif self.is_mutable or self.is_memory or self.is_temp_disk: + elif self.is_mutable or self.is_memory: return None elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): self.cached_hash = self.ds.inspect_data[self.name]["hash"] @@ -843,8 +672,6 @@ class Database: tags.append("mutable") if self.is_memory: tags.append("memory") - if self.is_temp_disk: - tags.append("temp_disk") if self.hash: tags.append(f"hash={self.hash}") if self.size is not None: @@ -855,21 +682,18 @@ class Database: return f"" -def _apply_write_wrapper(fn, wrapper_factory, track_event): +def _apply_write_wrapper(fn, wrapper_factory): """Apply a single write_wrapper context manager around fn. - ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally - ``track_event``, and returns a generator that yields exactly once. - Code before the yield runs before ``fn(conn)``, code after the yield - runs after. The result of ``fn(conn)`` is sent into the generator - via ``.send()``, and any exception raised by ``fn(conn)`` is thrown - via ``.throw()``. + ``wrapper_factory`` is a callable that takes ``(conn)`` and returns a + generator that yields exactly once. Code before the yield runs before + ``fn(conn)``, code after the yield runs after. The result of + ``fn(conn)`` is sent into the generator via ``.send()``, and any + exception raised by ``fn(conn)`` is thrown via ``.throw()``. """ def wrapped(conn): - gen = call_with_supported_arguments( - wrapper_factory, conn=conn, track_event=track_event - ) + gen = wrapper_factory(conn) # Advance to the yield point (run "before" code) try: next(gen) @@ -900,45 +724,16 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: - __slots__ = ( - "fn", - "task_id", - "loop", - "reply_future", - "isolated_connection", - "transaction", - ) + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") - def __init__( - self, fn, task_id, loop, reply_future, isolated_connection, transaction - ): + def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): self.fn = fn self.task_id = task_id - self.loop = loop - self.reply_future = reply_future + self.reply_queue = reply_queue self.isolated_connection = isolated_connection self.transaction = transaction -def _deliver_write_result(task, result, exception): - # Called from the write thread. Delivers the result back to the - # awaiting coroutine on its event loop via call_soon_threadsafe. - def _set(): - if task.reply_future.done(): - # Awaiter was cancelled; nothing to do. - return - if exception is not None: - task.reply_future.set_exception(exception) - else: - task.reply_future.set_result(result) - - try: - task.loop.call_soon_threadsafe(_set) - except RuntimeError: - # Event loop has been closed; the awaiter is gone. - pass - - class QueryInterrupted(Exception): def __init__(self, e, sql, params): self.e = e diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 2f78570b..149a4e5f 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -48,26 +48,12 @@ def register_actions(): resource_class=DatabaseResource, also_requires="view-database", ), - Action( - name="execute-write-sql", - abbr="ews", - description="Execute writable SQL queries", - resource_class=DatabaseResource, - also_requires="view-database", - ), Action( name="create-table", abbr="ct", description="Create tables", resource_class=DatabaseResource, ), - Action( - name="store-query", - abbr="sq", - description="Create stored queries", - resource_class=DatabaseResource, - also_requires="execute-sql", - ), # Table-level actions (child-level) Action( name="view-table", @@ -118,16 +104,4 @@ def register_actions(): description="View named query results", resource_class=QueryResource, ), - Action( - name="update-query", - abbr="uq", - description="Update stored queries", - resource_class=QueryResource, - ), - Action( - name="delete-query", - abbr="dq", - description="Delete stored queries", - resource_class=QueryResource, - ), ) diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py deleted file mode 100644 index e0cb3cdf..00000000 --- a/datasette/default_database_actions.py +++ /dev/null @@ -1,24 +0,0 @@ -from datasette import hookimpl -from datasette.resources import DatabaseResource - - -@hookimpl -def database_actions(datasette, actor, database, request): - async def inner(): - if not datasette.get_database(database).is_mutable: - return [] - if not await datasette.allowed( - action="execute-write-sql", - resource=DatabaseResource(database), - actor=actor, - ): - return [] - return [ - { - "href": datasette.urls.database(database) + "/-/execute-write", - "label": "Execute write SQL", - "description": "Run writable SQL with table permission checks.", - } - ] - - return inner diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py deleted file mode 100644 index 6127b2a6..00000000 --- a/datasette/default_debug_menu.py +++ /dev/null @@ -1,75 +0,0 @@ -from datasette import hookimpl -from datasette.jump import JumpSQL - -DEBUG_MENU_ITEMS = ( - ( - "/-/databases", - "Databases", - "List of databases known to this Datasette instance.", - ), - ( - "/-/plugins", - "Installed plugins", - "Review loaded plugins, their versions and their registered hooks.", - ), - ( - "/-/versions", - "Version info", - "Check the Python, SQLite and dependency versions used by this server.", - ), - ( - "/-/settings", - "Settings", - "Inspect the active Datasette settings and configuration values.", - ), - ( - "/-/permissions", - "Debug permissions", - "Test permission checks for actors, actions and resources.", - ), - ( - "/-/messages", - "Debug messages", - "Try out temporary flash messages shown to users.", - ), - ( - "/-/allow-debug", - "Debug allow rules", - "Explore how allow blocks match actors against permission rules.", - ), - ( - "/-/threads", - "Debug threads", - "Inspect worker threads and database tasks.", - ), - ( - "/-/actor", - "Debug actor", - "View the actor object for the current signed-in user.", - ), - ( - "/-/patterns", - "Pattern portfolio", - "Browse Datasette UI patterns.", - ), -) - - -@hookimpl -def jump_items_sql(datasette, actor, request): - async def inner(): - if not await datasette.allowed(action="debug-menu", actor=actor): - return [] - - return [ - JumpSQL.menu_item( - label=label, - url=datasette.urls.path(path), - description=description, - search_text=f"debug {label} {description}", - item_type="debug", - ) - for path, label, description in DEBUG_MENU_ITEMS - ] - - return inner diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py deleted file mode 100644 index d215e7ec..00000000 --- a/datasette/default_jump_items.py +++ /dev/null @@ -1,82 +0,0 @@ -from datasette import hookimpl -from datasette.jump import JumpSQL - - -@hookimpl -def jump_items_sql(datasette, actor, request): - async def inner(): - database_sql, database_params = await datasette.allowed_resources_sql( - action="view-database", actor=actor - ) - table_sql, table_params = await datasette.allowed_resources_sql( - action="view-table", actor=actor - ) - query_sql, query_params = await datasette.allowed_resources_sql( - action="view-query", actor=actor - ) - return [ - JumpSQL( - sql=f""" - WITH allowed_databases AS ( - {database_sql} - ) - SELECT - 'database' AS type, - parent AS label, - NULL AS description, - json_object( - 'method', 'database', - 'database', parent - ) AS url, - parent AS search_text, - NULL AS display_name - FROM allowed_databases - """, - params=database_params, - ), - JumpSQL( - sql=f""" - WITH allowed_tables AS ( - {table_sql} - ) - SELECT - CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, - allowed_tables.parent || ': ' || allowed_tables.child AS label, - NULL AS description, - json_object( - 'method', 'table', - 'database', allowed_tables.parent, - 'table', allowed_tables.child - ) AS url, - allowed_tables.parent || ' ' || allowed_tables.child AS search_text, - NULL AS display_name - FROM allowed_tables - LEFT JOIN catalog_views - ON catalog_views.database_name = allowed_tables.parent - AND catalog_views.view_name = allowed_tables.child - """, - params=table_params, - ), - JumpSQL( - sql=f""" - WITH allowed_queries AS ( - {query_sql} - ) - SELECT - 'query' AS type, - allowed_queries.parent || ': ' || allowed_queries.child AS label, - NULL AS description, - json_object( - 'method', 'query', - 'database', allowed_queries.parent, - 'query', allowed_queries.child - ) AS url, - allowed_queries.parent || ' ' || allowed_queries.child AS search_text, - NULL AS display_name - FROM allowed_queries - """, - params=query_params, - ), - ] - - return inner diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py new file mode 100644 index 00000000..85032387 --- /dev/null +++ b/datasette/default_menu_links.py @@ -0,0 +1,41 @@ +from datasette import hookimpl + + +@hookimpl +def menu_links(datasette, actor): + async def inner(): + if not await datasette.allowed(action="debug-menu", actor=actor): + return [] + + return [ + {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, + { + "href": datasette.urls.path("/-/plugins"), + "label": "Installed plugins", + }, + { + "href": datasette.urls.path("/-/versions"), + "label": "Version info", + }, + { + "href": datasette.urls.path("/-/settings"), + "label": "Settings", + }, + { + "href": datasette.urls.path("/-/permissions"), + "label": "Debug permissions", + }, + { + "href": datasette.urls.path("/-/messages"), + "label": "Debug messages", + }, + { + "href": datasette.urls.path("/-/allow-debug"), + "label": "Debug allow rules", + }, + {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"}, + {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, + {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, + ] + + return inner diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 6cd46f04..4ebe6147 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,6 +17,13 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from datasette.app import Datasette + +from datasette import hookimpl + # Re-export all hooks and public utilities from .restrictions import ( actor_restrictions_sql as actor_restrictions_sql, @@ -26,9 +33,26 @@ from .restrictions import ( from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql from .defaults import ( - # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, default_action_permissions_sql as default_action_permissions_sql, - default_query_permissions_sql as default_query_permissions_sql, DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, ) + + +@hookimpl +def skip_csrf(scope) -> Optional[bool]: + """Skip CSRF check for JSON content-type requests.""" + if scope["type"] == "http": + headers = scope.get("headers") or {} + if dict(headers).get(b"content-type") == b"application/json": + return True + return None + + +@hookimpl +def canned_queries(datasette: "Datasette", database: str, actor) -> dict: + """Return canned queries defined in datasette.yaml configuration.""" + queries = ( + ((datasette.config or {}).get("databases") or {}).get(database) or {} + ).get("queries") or {} + return queries diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 5bc74425..4c74219d 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -67,48 +67,3 @@ async def default_action_permissions_sql( return PermissionSQL.allow(reason=reason) return None - - -@hookimpl(specname="permission_resources_sql") -async def default_query_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], - action: str, -) -> Optional[PermissionSQL]: - actor_id = actor.get("id") if isinstance(actor, dict) else None - - if action not in {"view-query", "update-query", "delete-query"}: - return None - - params = {"query_owner_id": actor_id} - rule_sqls = [] - if actor_id is not None: - if action in {"update-query", "delete-query"}: - # Query owner can update/delete query - rule_sqls.append(""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE source = 'user' - AND owner_id = :query_owner_id - """) - else: - # Query owner can view-query - rule_sqls.append(""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE owner_id = :query_owner_id - """) - - # restriction_sql enforces private queries ONLY visible/mutable by owner - return PermissionSQL( - sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, - restriction_sql=""" - SELECT database_name AS parent, name AS child - FROM queries - WHERE is_private = 0 - OR owner_id = :query_owner_id - """, - params=params, - ) diff --git a/datasette/events.py b/datasette/events.py index e8786da9..5cd5ba3d 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -199,27 +199,6 @@ class UpdateRowEvent(Event): pks: list -@dataclass -class RenameTableEvent(Event): - """ - Event name: ``rename-table`` - - A table has been renamed. - - :ivar database: The name of the database containing the renamed table. - :type database: str - :ivar old_table: The previous name of the table. - :type old_table: str - :ivar new_table: The new name of the table. - :type new_table: str - """ - - name = "rename-table" - database: str - old_table: str - new_table: str - - @dataclass class DeleteRowEvent(Event): """ @@ -240,42 +219,6 @@ class DeleteRowEvent(Event): pks: list -@hookimpl -def write_wrapper(datasette, database, request, transaction): - def wrapper(conn, track_event): - # Snapshot rootpage -> name before the write - before = { - row[1]: row[0] - for row in conn.execute( - "select name, rootpage from sqlite_master" - " where type='table' and rootpage != 0" - ).fetchall() - } - yield - # Snapshot rootpage -> name after the write - after = { - row[1]: row[0] - for row in conn.execute( - "select name, rootpage from sqlite_master" - " where type='table' and rootpage != 0" - ).fetchall() - } - # Detect renames: same rootpage, different name - for rootpage, old_name in before.items(): - new_name = after.get(rootpage) - if new_name and new_name != old_name: - track_event( - RenameTableEvent( - actor=request.actor if request else None, - database=database, - old_table=old_name, - new_table=new_name, - ) - ) - - return wrapper - - @hookimpl def register_events(): return [ @@ -284,7 +227,6 @@ def register_events(): CreateTableEvent, CreateTokenEvent, AlterTableEvent, - RenameTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..bc4b6904 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -83,7 +83,7 @@ class Facet: self.ds = ds self.request = request self.database = database - # For foreign key expansion. Can be None for e.g. stored SQL queries: + # For foreign key expansion. Can be None for e.g. canned SQL queries: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] diff --git a/datasette/fixtures.py b/datasette/fixtures.py deleted file mode 100644 index 7c85e16a..00000000 --- a/datasette/fixtures.py +++ /dev/null @@ -1,415 +0,0 @@ -from datasette.utils.sqlite import sqlite3 -from datasette.utils import documented -import itertools -import random -import string - -__all__ = [ - "EXTRA_DATABASE_SQL", - "TABLES", - "TABLE_PARAMETERIZED_SQL", - "generate_compound_rows", - "generate_sortable_rows", - "populate_extra_database", - "populate_fixture_database", - "write_extra_database", - "write_fixture_database", -] - - -def generate_compound_rows(num): - """Generate rows for the compound_three_primary_keys fixture table.""" - for a, b, c in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=3), num - ): - yield a, b, c, f"{a}-{b}-{c}" - - -def generate_sortable_rows(num): - """Generate rows for the sortable fixture table.""" - rand = random.Random(42) - for a, b in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=2), num - ): - yield { - "pk1": a, - "pk2": b, - "content": f"{a}-{b}", - "sortable": rand.randint(-100, 100), - "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), - "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), - "text": rand.choice(["$null", "$blah"]), - } - - -TABLES = ( - """ -CREATE TABLE simple_primary_key ( - id integer primary key, - content text -); - -CREATE TABLE primary_key_multiple_columns ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE primary_key_multiple_columns_explicit_label ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE compound_primary_key ( - pk1 varchar(30), - pk2 varchar(30), - content text, - PRIMARY KEY (pk1, pk2) -); - -INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); -INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); -INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); - -CREATE TABLE compound_three_primary_keys ( - pk1 varchar(30), - pk2 varchar(30), - pk3 varchar(30), - content text, - PRIMARY KEY (pk1, pk2, pk3) -); -CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); - -CREATE TABLE foreign_key_references ( - pk varchar(30) primary key, - foreign_key_with_label integer, - foreign_key_with_blank_label integer, - foreign_key_with_no_label varchar(30), - foreign_key_compound_pk1 varchar(30), - foreign_key_compound_pk2 varchar(30), - FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) - FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) -); - -CREATE TABLE sortable ( - pk1 varchar(30), - pk2 varchar(30), - content text, - sortable integer, - sortable_with_nulls real, - sortable_with_nulls_2 real, - text text, - PRIMARY KEY (pk1, pk2) -); - -CREATE TABLE no_primary_key ( - content text, - a text, - b text, - c text -); - -CREATE TABLE [123_starts_with_digits] ( - content text -); - -CREATE VIEW paginated_view AS - SELECT - content, - '- ' || content || ' -' AS content_extra - FROM no_primary_key; - -CREATE TABLE "Table With Space In Name" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "table/with/slashes.csv" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "complex_foreign_keys" ( - pk varchar(30) primary key, - f1 integer, - f2 integer, - f3 integer, - FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) -); - -CREATE TABLE "custom_foreign_key_label" ( - pk varchar(30) primary key, - foreign_key_with_custom_label text, - FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) -); - -CREATE TABLE tags ( - tag TEXT PRIMARY KEY -); - -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text, - [name with . and spaces] text -); - -CREATE TABLE searchable_tags ( - searchable_id integer, - tag text, - PRIMARY KEY (searchable_id, tag), - FOREIGN KEY (searchable_id) REFERENCES searchable(pk), - FOREIGN KEY (tag) REFERENCES tags(tag) -); - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); - -INSERT INTO tags VALUES ("canine"); -INSERT INTO tags VALUES ("feline"); - -INSERT INTO searchable_tags (searchable_id, tag) VALUES - (1, "feline"), - (2, "canine") -; - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); -INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); - -CREATE TABLE [select] ( - [group] text, - [having] text, - [and] text, - [json] text -); -INSERT INTO [select] VALUES ('group', 'having', 'and', - '{"href": "http://example.com/", "label":"Example"}' -); - -CREATE TABLE infinity ( - value REAL -); -INSERT INTO infinity VALUES - (1e999), - (-1e999), - (1.5) -; - -CREATE TABLE facet_cities ( - id integer primary key, - name text -); -INSERT INTO facet_cities (id, name) VALUES - (1, 'San Francisco'), - (2, 'Los Angeles'), - (3, 'Detroit'), - (4, 'Memnonia') -; - -CREATE TABLE facetable ( - pk integer primary key, - created text, - planet_int integer, - on_earth integer, - state text, - _city_id integer, - _neighborhood text, - tags text, - complex_array text, - distinct_some_null, - n text, - FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) -); -INSERT INTO facetable - (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) -VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) -; - -CREATE TABLE binary_data ( - data BLOB -); - --- Many 2 Many demo: roadside attractions! - -CREATE TABLE roadside_attractions ( - pk integer primary key, - name text, - address text, - url text, - latitude real, - longitude real -); -INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", - 37.0167, -122.0024 -); -INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", - 37.3184, -121.9511 -); -INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, - 37.5793, -122.3442 -); -INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", - 37.0414, -122.0725 -); - -CREATE TABLE attraction_characteristic ( - pk integer primary key, - name text -); -INSERT INTO attraction_characteristic VALUES ( - 1, "Museum" -); -INSERT INTO attraction_characteristic VALUES ( - 2, "Paranormal" -); - -CREATE TABLE roadside_attraction_characteristics ( - attraction_id INTEGER REFERENCES roadside_attractions(pk), - characteristic_id INTEGER REFERENCES attraction_characteristic(pk) -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 1, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 2, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 3, 1 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 1 -); - -INSERT INTO simple_primary_key VALUES (1, 'hello'); -INSERT INTO simple_primary_key VALUES (2, 'world'); -INSERT INTO simple_primary_key VALUES (3, ''); -INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); -INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); - -INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); -INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); - -INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); -INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); - -INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); -INSERT INTO custom_foreign_key_label VALUES (1, 1); - -INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); - -CREATE VIEW simple_view AS - SELECT content, upper(content) AS upper_content FROM simple_primary_key; - -CREATE VIEW searchable_view AS - SELECT * from searchable; - -CREATE VIEW searchable_view_configured_by_metadata AS - SELECT * from searchable; - -""" - + "\n".join( - [ - 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( - i=i + 1 - ) - for i in range(201) - ] - ) - + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' - + "\n".join( - [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) - for a, b, c, content in generate_compound_rows(1001) - ] - ) - + "\n".join(["""INSERT INTO sortable VALUES ( - "{pk1}", "{pk2}", "{content}", {sortable}, - {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); - """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) -) - -TABLE_PARAMETERIZED_SQL = [ - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (null);", []), -] - -EXTRA_DATABASE_SQL = """ -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text -); - -CREATE VIEW searchable_view AS SELECT * FROM searchable; - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS3 (text1, text2, content="searchable"); -INSERT INTO "searchable_fts" (rowid, text1, text2) - SELECT rowid, text1, text2 FROM searchable; -""" - - -@documented(label="datasette_fixtures_populate_fixture_database") -def populate_fixture_database(conn): - """Populate a SQLite connection with Datasette's test fixture tables.""" - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - - -def populate_extra_database(conn): - """Populate a SQLite connection with the extra database used in tests.""" - conn.executescript(EXTRA_DATABASE_SQL) - - -def write_fixture_database(db_filename): - """Write Datasette's test fixture tables to a SQLite database file.""" - conn = sqlite3.connect(db_filename) - try: - populate_fixture_database(conn) - finally: - conn.close() - - -def write_extra_database(db_filename): - """Write the extra test database tables to a SQLite database file.""" - conn = sqlite3.connect(db_filename) - try: - populate_extra_database(conn) - finally: - conn.close() diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index dcd502af..2ab9d0c5 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -137,6 +137,11 @@ def permission_resources_sql(datasette, actor, action): """ +@hookspec +def canned_queries(datasette, database, actor): + """Return a dictionary of canned query definitions or an awaitable function that returns them""" + + @hookspec def register_magic_parameters(datasette): """Return a list of (name, function) magic parameter functions""" @@ -152,11 +157,6 @@ def menu_links(datasette, actor, request): """Links for the navigation menu""" -@hookspec -def jump_items_sql(datasette, actor, request): - """SQL fragments for extra items in the jump menu""" - - @hookspec def row_actions(datasette, actor, request, database, table, row): """Links for the row actions menu""" @@ -174,7 +174,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and stored query actions menu""" + """Links for the query and canned query actions menu""" @hookspec @@ -187,6 +187,11 @@ def homepage_actions(datasette, actor, request): """Links for the homepage actions menu""" +@hookspec +def skip_csrf(datasette, scope): + """Mechanism for skipping CSRF checks for certain requests""" + + @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" @@ -228,8 +233,8 @@ def top_query(datasette, request, database, sql): @hookspec -def top_stored_query(datasette, request, database, query_name): - """HTML to include at the top of the stored query page""" +def top_canned_query(datasette, request, database, query_name): + """HTML to include at the top of the canned query page""" @hookspec @@ -241,18 +246,12 @@ def register_token_handler(datasette): def write_wrapper(datasette, database, request, transaction): """Called when a write function is about to execute. - Return a generator function that accepts a ``conn`` argument and - optionally a ``track_event`` argument. The generator should - ``yield`` exactly once: code before the ``yield`` runs before - the write, code after the ``yield`` runs after the write - completes. The result of the write is sent back through the - ``yield``, so you can capture it with ``result = yield``. - - If your generator accepts ``track_event``, you can call - ``track_event(event)`` to queue an event that will be dispatched - via ``datasette.track_event()`` after the write commits - successfully. Events are discarded if the write raises an - exception. + Return a generator function that accepts a ``conn`` argument. + The generator should ``yield`` exactly once: code before the + ``yield`` runs before the write, code after the ``yield`` runs + after the write completes. The result of the write is sent + back through the ``yield``, so you can capture it with + ``result = yield``. If the write raises an exception, it is thrown into the generator so you can handle it with a try/except around the ``yield``. diff --git a/datasette/jump.py b/datasette/jump.py deleted file mode 100644 index d138e827..00000000 --- a/datasette/jump.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Any - - -@dataclass -class JumpSQL: - sql: str - params: dict[str, Any] | None = None - database: str | None = None - - @classmethod - def menu_item( - cls, - *, - label: str, - url: str, - description: str = "Menu item", - search_text: str | None = None, - display_name: str | None = None, - item_type: str = "menu", - ) -> "JumpSQL": - if search_text is None: - search_text = " ".join( - text for text in (label, display_name, description) if text is not None - ) - return cls( - sql=""" - SELECT - :type AS type, - :label AS label, - :description AS description, - :url AS url, - :search_text AS search_text, - :display_name AS display_name - """, - params={ - "type": item_type, - "label": label, - "description": description, - "url": url, - "search_text": search_text, - "display_name": display_name, - }, - ) - - -_PARAM_RE = re.compile(r"(? str: + def resources_sql(cls, datasette, actor=None) -> str: """ Return SQL query that returns all resources of this type. diff --git a/datasette/plugins.py b/datasette/plugins.py index 5a31cdad..b01b386c 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -28,9 +28,7 @@ DEFAULT_PLUGINS = ( "datasette.default_column_types", "datasette.default_magic_parameters", "datasette.blob_renderer", - "datasette.default_debug_menu", - "datasette.default_jump_items", - "datasette.default_database_actions", + "datasette.default_menu_links", "datasette.handle_exception", "datasette.forbidden", "datasette.events", diff --git a/datasette/resources.py b/datasette/resources.py index ee2e6d98..236b3598 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -41,7 +41,7 @@ class TableResource(Resource): class QueryResource(Resource): - """A stored query in a database.""" + """A canned query in a database.""" name = "query" parent_class = DatabaseResource @@ -51,8 +51,42 @@ class QueryResource(Resource): @classmethod async def resources_sql(cls, datasette, actor=None) -> str: - return """ - SELECT q.database_name AS parent, q.name AS child - FROM queries q - JOIN catalog_databases cd ON cd.database_name = q.database_name - """ + from datasette.plugins import pm + from datasette.utils import await_me_maybe + + # Get all databases from catalog + db = datasette.get_internal_database() + result = await db.execute("SELECT database_name FROM catalog_databases") + databases = [row[0] for row in result.rows] + + # Gather canned queries for this actor from all databases. + # This keeps allowed_resources("view-query", actor=...) consistent with + # actor-specific canned_queries() implementations. + query_pairs = [] + for database_name in databases: + # Call the hook to get queries (including from config via default plugin) + for queries_result in pm.hook.canned_queries( + datasette=datasette, + database=database_name, + actor=actor, + ): + queries = await await_me_maybe(queries_result) + if queries: + for query_name in queries.keys(): + query_pairs.append((database_name, query_name)) + + # Build SQL + if not query_pairs: + return "SELECT NULL AS parent, NULL AS child WHERE 0" + + # Generate UNION ALL query + selects = [] + for db_name, query_name in query_pairs: + # Escape single quotes by doubling them + db_escaped = db_name.replace("'", "''") + query_escaped = query_name.replace("'", "''") + selects.append( + f"SELECT '{db_escaped}' AS parent, '{query_escaped}' AS child" + ) + + return " UNION ALL ".join(selects) diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..0a6efd4c 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -362,32 +362,6 @@ form.nav-menu-logout { .nav-menu-inner a { display: block; } -.nav-menu-inner button.button-as-link { - display: block; - width: 100%; - text-align: left; - font: inherit; -} -.nav-menu-inner .keyboard-shortcut { - float: right; - box-sizing: border-box; - min-width: 1.4em; - margin-left: 0.75rem; - padding: 0 0.35em; - border: 1px solid rgba(255,255,244,0.6); - border-radius: 3px; - background: rgba(255,255,244,0.12); - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 0.85em; - line-height: 1.35; - text-align: center; - text-decoration: none; -} -@media (max-width: 640px) { - .nav-menu-inner .keyboard-shortcut { - display: none; - } -} /* Table/database actions menu */ .page-action-menu { @@ -844,8 +818,7 @@ dialog.mobile-column-actions-dialog::backdrop { } .mobile-column-actions-dialog .list-wrap { - flex: 1 1 auto; - min-height: 0; + flex: 1; overflow-y: auto; overflow-x: hidden; position: relative; @@ -1013,180 +986,6 @@ dialog.mobile-column-actions-dialog::backdrop { color: var(--ink); } -dialog.set-column-type-dialog { - --ink: #0f0f0f; - --paper: #f5f3ef; - --muted: #6b6b6b; - --rule: #e2dfd8; - --accent: #1a56db; - --card: #ffffff; - border: none; - border-radius: var(--modal-border-radius, 0.75rem); - padding: 0; - margin: auto; - width: min(520px, calc(100vw - 32px)); - max-width: 95vw; - max-height: min(720px, calc(100vh - 32px)); - box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); - animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; - overflow: hidden; - font-family: system-ui, -apple-system, sans-serif; - background: var(--card); -} - -dialog.set-column-type-dialog[open] { - display: flex; - flex-direction: column; -} - -dialog.set-column-type-dialog::backdrop { - background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); - backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); - animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; -} - -.set-column-type-dialog .modal-header { - padding: 20px 24px 12px; - border-bottom: 1px solid var(--rule); - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - flex-shrink: 0; -} - -.set-column-type-dialog .modal-title { - font-size: 1rem; - font-weight: 600; - color: var(--ink); -} - -.set-column-type-dialog .modal-meta { - font-family: ui-monospace, monospace; - font-size: 0.7rem; - color: var(--muted); - background: var(--paper); - padding: 3px 9px; - border-radius: 20px; -} - -.set-column-type-status, -.set-column-type-empty, -.set-column-type-error { - margin: 0; - padding: 12px 24px 0; -} - -.set-column-type-status, -.set-column-type-empty { - color: var(--muted); - font-size: 0.9rem; -} - -.set-column-type-error { - color: #b91c1c; - font-size: 0.9rem; -} - -.set-column-type-options { - padding: 16px 24px 24px; - overflow-y: auto; - display: grid; - gap: 12px; -} - -.set-column-type-option { - display: grid; - grid-template-columns: auto 1fr; - gap: 12px; - align-items: start; - padding: 14px 16px; - border: 1px solid var(--rule); - border-radius: 8px; - background: #fcfbf9; - cursor: pointer; -} - -.set-column-type-option:focus-within { - border-color: var(--accent); - box-shadow: 0 0 0 3px rgba(26, 86, 219, 0.12); -} - -.set-column-type-option input { - margin-top: 3px; -} - -.set-column-type-option-content { - display: grid; - gap: 4px; -} - -.set-column-type-option-name { - font-family: ui-monospace, monospace; - font-size: 0.95rem; - color: var(--ink); -} - -.set-column-type-option-description { - color: var(--muted); - font-size: 0.9rem; -} - -.set-column-type-dialog .modal-footer { - padding: 14px 20px; - border-top: 1px solid var(--rule); - display: flex; - align-items: center; - gap: 10px; - flex-shrink: 0; - background: var(--paper); -} - -.set-column-type-dialog .footer-info { - flex: 1; - font-family: ui-monospace, monospace; - font-size: 0.68rem; - color: var(--muted); -} - -.set-column-type-dialog .btn { - border: none; - border-radius: 5px; - padding: 9px 20px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - touch-action: manipulation; - font-family: inherit; - transition: background 0.12s; -} - -.set-column-type-dialog .btn-ghost { - background: transparent; - color: var(--muted); - border: 1px solid var(--rule); -} - -.set-column-type-dialog .btn-ghost:hover { - background: var(--rule); - color: var(--ink); -} - -.set-column-type-dialog .btn-primary { - background: var(--accent); - color: #fff; -} - -.set-column-type-dialog .btn-primary:hover { - background: #1949b8; -} - -.set-column-type-dialog .btn:disabled { - opacity: 0.65; - cursor: wait; -} - @media (max-width: 640px) { dialog.mobile-column-actions-dialog { width: 95vw; @@ -1219,21 +1018,6 @@ dialog.set-column-type-dialog::backdrop { padding-left: 18px; padding-right: 18px; } - - dialog.set-column-type-dialog { - width: 95vw; - max-height: 85vh; - border-radius: 0.5rem; - } - - .set-column-type-dialog .modal-header, - .set-column-type-status, - .set-column-type-empty, - .set-column-type-error, - .set-column-type-options { - padding-left: 18px; - padding-right: 18px; - } } @media only screen and (max-width: 576px) { @@ -1409,15 +1193,11 @@ svg.dropdown-menu-icon { border-bottom: 5px solid #666; } -.stored-query-edit-sql { +.canned-query-edit-sql { padding-left: 0.5em; position: relative; top: 1px; } -.save-query { - display: inline-block; - margin-left: 0.45em; -} .blob-download { display: block; diff --git a/datasette/static/datasette-manager.js b/datasette/static/datasette-manager.js index e75f7aae..d2347ab3 100644 --- a/datasette/static/datasette-manager.js +++ b/datasette/static/datasette-manager.js @@ -82,19 +82,6 @@ const datasetteManager = { return columnActions; }, - makeJumpSections: (context) => { - let jumpSections = []; - - datasetteManager.plugins.forEach((plugin) => { - if (plugin.makeJumpSections) { - const sections = plugin.makeJumpSections(context) || []; - jumpSections.push(...sections); - } - }); - - return jumpSections; - }, - /** * In MVP, each plugin can only have 1 instance. * In future, panels could be repeated. We omit that for now since so many plugins depend on @@ -205,6 +192,7 @@ const initializeDatasette = () => { // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window. window.__DATASETTE__ = datasetteManager; + console.debug("Datasette Manager Created!"); const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, { detail: datasetteManager, diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index ec2d23d8..95e7dfc5 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -1,22 +1,10 @@ -let navigationSearchInstanceCounter = 0; - class NavigationSearch extends HTMLElement { constructor() { super(); - this.instanceId = ++navigationSearchInstanceCounter; - this.inputId = `navigation-search-input-${this.instanceId}`; - this.instructionsId = `navigation-search-instructions-${this.instanceId}`; - this.listboxId = `navigation-search-results-${this.instanceId}`; - this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; - this.statusId = `navigation-search-status-${this.instanceId}`; - this.titleId = `navigation-search-title-${this.instanceId}`; this.attachShadow({ mode: "open" }); this.selectedIndex = -1; this.matches = []; - this.renderedMatches = []; this.debounceTimer = null; - this.restoreFocusTarget = null; - this.shouldRestoreFocus = true; this.render(); this.setupEventListeners(); @@ -66,20 +54,16 @@ class NavigationSearch extends HTMLElement { .search-container { display: flex; flex-direction: column; + height: 100%; } .search-input-wrapper { padding: 1.25rem; border-bottom: 1px solid #e5e7eb; - display: flex; - gap: 0.5rem; - align-items: center; } .search-input { width: 100%; - flex: 1; - min-width: 0; padding: 0.75rem 1rem; font-size: 1rem; border: 2px solid #e5e7eb; @@ -93,36 +77,12 @@ class NavigationSearch extends HTMLElement { border-color: #2563eb; } - .close-search { - background: transparent; - border: 1px solid transparent; - border-radius: 0.375rem; - color: #4b5563; - cursor: pointer; - flex: 0 0 auto; - font: inherit; - font-size: 1.5rem; - height: 2.75rem; - line-height: 1; - width: 2.75rem; - } - - .close-search:hover, - .close-search:focus { - background-color: #f3f4f6; - border-color: #d1d5db; - } - .results-container { overflow-y: auto; height: calc(80vh - 180px); padding: 0.5rem; } - .results-list:empty { - display: none; - } - .result-item { padding: 0.875rem 1rem; cursor: pointer; @@ -141,81 +101,16 @@ class NavigationSearch extends HTMLElement { background-color: #dbeafe; } - .result-item > div { - flex: 1; - min-width: 0; - } - - .jump-start-content { - border-bottom: 1px solid #e5e7eb; - margin-bottom: 0.5rem; - padding: 0.5rem 0.5rem 1rem; - } - - .jump-start-content:empty { - display: none; - } - .result-name { font-weight: 500; color: #111827; } - .result-label { - font-size: 0.875rem; - color: #4b5563; - } - - .result-type { - color: #4b5563; - font-size: 0.75rem; - font-weight: 600; - text-transform: uppercase; - } - .result-url { font-size: 0.875rem; color: #6b7280; } - .result-description { - color: #374151; - display: -webkit-box; - font-size: 0.8125rem; - line-height: 1.35; - margin-top: 0.35rem; - overflow: hidden; - -webkit-box-orient: vertical; - -webkit-line-clamp: 2; - } - - .results-heading { - color: #4b5563; - font-size: 0.75rem; - font-weight: 600; - letter-spacing: 0; - padding: 0.5rem 1rem 0.25rem; - text-transform: uppercase; - } - - .recent-actions { - padding: 0.25rem 1rem 0.75rem; - } - - .clear-recent { - background: transparent; - border: 0; - color: #2563eb; - cursor: pointer; - font: inherit; - font-size: 0.875rem; - padding: 0; - } - - .clear-recent:hover { - text-decoration: underline; - } - .no-results { padding: 2rem; text-align: center; @@ -241,18 +136,6 @@ class NavigationSearch extends HTMLElement { font-family: monospace; } - .visually-hidden { - border: 0; - clip: rect(0 0 0 0); - height: 1px; - margin: -1px; - overflow: hidden; - padding: 0; - position: absolute; - white-space: nowrap; - width: 1px; - } - /* Mobile optimizations */ @media (max-width: 640px) { dialog { @@ -280,29 +163,19 @@ class NavigationSearch extends HTMLElement { } - +
- - -
-
-
+
↑ ↓ Navigate Enter Select @@ -316,7 +189,6 @@ class NavigationSearch extends HTMLElement { setupEventListeners() { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); - const closeButton = this.shadowRoot.querySelector(".close-search"); const resultsContainer = this.shadowRoot.querySelector(".results-container"); @@ -328,17 +200,6 @@ class NavigationSearch extends HTMLElement { } }); - document.addEventListener("click", (e) => { - const trigger = e.target.closest("[data-navigation-search-open]"); - if (trigger) { - e.preventDefault(); - const details = trigger.closest("details"); - const restoreTarget = details?.querySelector("summary") || trigger; - details?.removeAttribute("open"); - this.openMenu(restoreTarget); - } - }); - // Input event input.addEventListener("input", (e) => { this.handleSearch(e.target.value); @@ -360,19 +221,8 @@ class NavigationSearch extends HTMLElement { } }); - closeButton.addEventListener("click", () => { - this.closeMenu(); - }); - // Click on result item resultsContainer.addEventListener("click", (e) => { - const clearRecent = e.target.closest("[data-clear-recent-items]"); - if (clearRecent) { - e.preventDefault(); - this.clearRecentItems(); - return; - } - const item = e.target.closest(".result-item"); if (item) { const index = parseInt(item.dataset.index); @@ -387,15 +237,6 @@ class NavigationSearch extends HTMLElement { } }); - dialog.addEventListener("cancel", (e) => { - e.preventDefault(); - this.closeMenu(); - }); - - dialog.addEventListener("close", () => { - this.onMenuClosed(); - }); - // Initial load this.loadInitialData(); } @@ -410,106 +251,6 @@ class NavigationSearch extends HTMLElement { ); } - setElementAttribute(element, name, value) { - if (!element) { - return; - } - if (typeof element.setAttribute === "function") { - element.setAttribute(name, value); - } else { - element[name] = String(value); - } - } - - removeElementAttribute(element, name) { - if (!element) { - return; - } - if (typeof element.removeAttribute === "function") { - element.removeAttribute(name); - } else { - delete element[name]; - } - } - - focusRestoreTarget(trigger) { - if (trigger && typeof trigger.focus === "function") { - return trigger; - } - if ( - document.activeElement && - typeof document.activeElement.focus === "function" - ) { - return document.activeElement; - } - return null; - } - - setNavigationTriggersExpanded(expanded) { - if (typeof document.querySelectorAll !== "function") { - return; - } - document - .querySelectorAll("[data-navigation-search-open]") - .forEach((trigger) => { - this.setElementAttribute( - trigger, - "aria-expanded", - expanded ? "true" : "false", - ); - }); - } - - resultOptionId(index) { - return `${this.listboxId}-option-${index}`; - } - - updateComboboxState() { - const dialog = this.shadowRoot.querySelector("dialog"); - const input = this.shadowRoot.querySelector(".search-input"); - const matches = this.renderedMatches || []; - this.setElementAttribute( - input, - "aria-expanded", - dialog && dialog.open && matches.length > 0 ? "true" : "false", - ); - - if ( - dialog && - dialog.open && - this.selectedIndex >= 0 && - this.selectedIndex < matches.length - ) { - this.setElementAttribute( - input, - "aria-activedescendant", - this.resultOptionId(this.selectedIndex), - ); - } else { - this.removeElementAttribute(input, "aria-activedescendant"); - } - } - - setStatus(message) { - const status = this.shadowRoot.querySelector(`#${this.statusId}`); - if (status) { - status.textContent = message || ""; - } - } - - resultsStatus(count, truncated) { - if (truncated) { - return "More than 100 results. Keep typing to narrow the list."; - } - if (count === 0) { - return "No results found."; - } - if (count === 1) { - return "1 result."; - } - return `${count} results.`; - } - loadInitialData() { const itemsAttr = this.getAttribute("items"); if (itemsAttr) { @@ -526,11 +267,6 @@ class NavigationSearch extends HTMLElement { handleSearch(query) { clearTimeout(this.debounceTimer); - if (query.trim()) { - this.setStatus("Searching..."); - } else { - this.setStatus(""); - } this.debounceTimer = setTimeout(() => { const url = this.getAttribute("url"); @@ -553,262 +289,65 @@ class NavigationSearch extends HTMLElement { this.matches = data.matches || []; this.selectedIndex = this.matches.length > 0 ? 0 : -1; this.renderResults(); - if (query.trim()) { - this.setStatus(this.resultsStatus(this.matches.length, data.truncated)); - } else { - this.setStatus(""); - } } catch (e) { console.error("Failed to fetch search results:", e); this.matches = []; this.renderResults(); - this.setStatus("Search failed."); } } filterLocalItems(query) { if (!query.trim()) { - this.matches = this.allItems || []; + this.matches = []; } else { const lowerQuery = query.toLowerCase(); this.matches = (this.allItems || []).filter( (item) => item.name.toLowerCase().includes(lowerQuery) || - (item.display_name || "").toLowerCase().includes(lowerQuery) || item.url.toLowerCase().includes(lowerQuery), ); } this.selectedIndex = this.matches.length > 0 ? 0 : -1; this.renderResults(); - if (query.trim()) { - this.setStatus(this.resultsStatus(this.matches.length, false)); - } else { - this.setStatus(""); - } - } - - recentItemsStorageKey() { - return "datasette.navigationSearch.recentItems"; - } - - loadRecentItems() { - if (typeof localStorage === "undefined") { - return []; - } - - try { - const raw = localStorage.getItem(this.recentItemsStorageKey()); - if (!raw) { - return []; - } - const parsed = JSON.parse(raw); - if (!Array.isArray(parsed)) { - return []; - } - return parsed - .filter((item) => item && item.name && item.url) - .map((item) => ({ - name: String(item.name), - display_name: item.display_name ? String(item.display_name) : "", - url: String(item.url), - type: item.type ? String(item.type) : "", - description: item.description ? String(item.description) : "", - })) - .slice(0, 5); - } catch (e) { - return []; - } - } - - saveRecentItem(match) { - if ( - typeof localStorage === "undefined" || - !match || - !match.name || - !match.url - ) { - return; - } - - try { - const item = { - name: String(match.name), - display_name: match.display_name ? String(match.display_name) : "", - url: String(match.url), - type: match.type ? String(match.type) : "", - description: match.description ? String(match.description) : "", - }; - const recentItems = this.loadRecentItems().filter( - (recentItem) => recentItem.url !== item.url, - ); - localStorage.setItem( - this.recentItemsStorageKey(), - JSON.stringify([item, ...recentItems].slice(0, 5)), - ); - } catch (e) { - // localStorage may be unavailable, full, or disabled. - } - } - - clearRecentItems() { - if (typeof localStorage === "undefined") { - return; - } - - try { - localStorage.removeItem(this.recentItemsStorageKey()); - } catch (e) { - localStorage.setItem(this.recentItemsStorageKey(), "[]"); - } - this.renderResults(); - this.setStatus("Recent items cleared."); - } - - jumpSections() { - const manager = window.__DATASETTE__; - if (!manager || typeof manager.makeJumpSections !== "function") { - return []; - } - const sections = manager.makeJumpSections({ - navigationSearch: this, - }); - return Array.isArray(sections) - ? sections.filter( - (section) => section && typeof section.render === "function", - ) - : []; - } - - jumpSectionsHtml(jumpSections) { - return jumpSections - .map((section, index) => { - const id = section.id - ? ` data-jump-section-id="${this.escapeHtml(section.id)}"` - : ""; - return `
`; - }) - .join(""); - } - - renderJumpSections(container, jumpSections) { - jumpSections.forEach((section, index) => { - const node = container.querySelector( - `[data-jump-section-index="${index}"]`, - ); - if (!node) { - return; - } - section.render(node, { - navigationSearch: this, - container, - input: this.shadowRoot.querySelector(".search-input"), - }); - }); - } - - resultItemHtml(match, index) { - const displayName = match.display_name || match.name; - const label = - match.display_name && match.display_name !== match.name - ? `
${this.escapeHtml(match.name)}
` - : ""; - const type = match.type - ? `
${this.escapeHtml(match.type)}
` - : ""; - const description = match.description - ? `
${this.escapeHtml( - match.description, - )}
` - : ""; - return ` -
-
- ${type} -
${this.escapeHtml(displayName)}
- ${label} -
${this.escapeHtml(match.url)}
- ${description} -
-
- `; } renderResults() { const container = this.shadowRoot.querySelector(".results-container"); const input = this.shadowRoot.querySelector(".search-input"); - const showStartContent = !input.value.trim(); - const jumpSections = showStartContent ? this.jumpSections() : []; - const startBlock = showStartContent - ? this.jumpSectionsHtml(jumpSections) - : ""; - const recentItems = showStartContent ? this.loadRecentItems() : []; - const defaultMatches = showStartContent ? [] : this.matches; - const renderedMatches = [...recentItems, ...defaultMatches]; - this.renderedMatches = renderedMatches; - const emptyListbox = `
`; - if (renderedMatches.length) { - if ( - this.selectedIndex < 0 || - this.selectedIndex >= renderedMatches.length - ) { - this.selectedIndex = 0; - } - } else { - this.selectedIndex = -1; - } - - if (renderedMatches.length === 0) { - if (startBlock) { - container.innerHTML = startBlock + emptyListbox; - this.renderJumpSections(container, jumpSections); - } else if (showStartContent) { - container.innerHTML = emptyListbox; - } else { - const message = input.value.trim() - ? "No results found" - : "Start typing to search..."; - container.innerHTML = `${emptyListbox}
${message}
`; - } - this.updateComboboxState(); + if (this.matches.length === 0) { + const message = input.value.trim() + ? "No results found" + : "Start typing to search..."; + container.innerHTML = `
${message}
`; return; } - const recentHeading = recentItems.length - ? `
Recent
` - : ""; - const recentGroup = recentItems.length - ? `
${recentItems - .map((match, index) => this.resultItemHtml(match, index)) - .join("")}
` - : ""; - const recentActions = recentItems.length - ? `
` - : ""; - const defaultHtml = defaultMatches - .map((match, index) => - this.resultItemHtml(match, recentItems.length + index), + container.innerHTML = this.matches + .map( + (match, index) => ` +
+
+
${this.escapeHtml( + match.name, + )}
+
${this.escapeHtml(match.url)}
+
+
+ `, ) .join(""); - container.innerHTML = - startBlock + - recentHeading + - `
${recentGroup}${defaultHtml}
` + - recentActions; - this.renderJumpSections(container, jumpSections); - this.updateComboboxState(); // Scroll selected item into view if (this.selectedIndex >= 0) { - const selectedItem = container.querySelector( - `.result-item[data-index="${this.selectedIndex}"]`, - ); + const selectedItem = container.children[this.selectedIndex]; if (selectedItem) { selectedItem.scrollIntoView({ block: "nearest" }); } @@ -816,27 +355,22 @@ class NavigationSearch extends HTMLElement { } moveSelection(direction) { - const matches = this.renderedMatches || this.matches; const newIndex = this.selectedIndex + direction; - if (newIndex >= 0 && newIndex < matches.length) { + if (newIndex >= 0 && newIndex < this.matches.length) { this.selectedIndex = newIndex; this.renderResults(); } } selectCurrentItem() { - const matches = this.renderedMatches || this.matches; - if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) { + if (this.selectedIndex >= 0 && this.selectedIndex < this.matches.length) { this.selectItem(this.selectedIndex); } } selectItem(index) { - const matches = this.renderedMatches || this.matches; - const match = matches[index]; + const match = this.matches[index]; if (match) { - this.saveRecentItem(match); - // Dispatch custom event this.dispatchEvent( new CustomEvent("select", { @@ -849,59 +383,32 @@ class NavigationSearch extends HTMLElement { // Navigate to URL window.location.href = match.url; - this.closeMenu({ restoreFocus: false }); + this.closeMenu(); } } - openMenu(trigger) { + openMenu() { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); - this.restoreFocusTarget = this.focusRestoreTarget(trigger); - this.shouldRestoreFocus = true; - if (!dialog.open) { - dialog.showModal(); - } - this.setNavigationTriggersExpanded(true); + dialog.showModal(); input.value = ""; input.focus(); - // Reset state, then populate the default jump list. + // Reset state - start with no items shown this.matches = []; this.selectedIndex = -1; this.renderResults(); - this.setStatus(""); } - closeMenu(options = {}) { + closeMenu() { const dialog = this.shadowRoot.querySelector("dialog"); - this.shouldRestoreFocus = options.restoreFocus !== false; - if (dialog.open) { - dialog.close(); - } else { - this.onMenuClosed(); - } - } - - onMenuClosed() { - const input = this.shadowRoot.querySelector(".search-input"); - this.setElementAttribute(input, "aria-expanded", "false"); - this.removeElementAttribute(input, "aria-activedescendant"); - this.setNavigationTriggersExpanded(false); - this.setStatus(""); - if ( - this.shouldRestoreFocus && - this.restoreFocusTarget && - typeof this.restoreFocusTarget.focus === "function" - ) { - this.restoreFocusTarget.focus(); - } - this.restoreFocusTarget = null; + dialog.close(); } escapeHtml(text) { const div = document.createElement("div"); - div.textContent = text == null ? "" : text; + div.textContent = text; return div.innerHTML; } } diff --git a/datasette/static/table.js b/datasette/static/table.js index e9115453..1e243703 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -10,9 +10,6 @@ var DROPDOWN_ICON_SVG = ` `; -var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog"; -var setColumnTypeDialogState = null; - function getParams() { return new URLSearchParams(location.search); } @@ -102,259 +99,6 @@ function getColumnTypeText(th) { return `Type: ${columnType.toUpperCase()}${notNull}`; } -function getSetColumnTypeData() { - return window._setColumnTypeData || null; -} - -function getSetColumnTypeConfig(column) { - var data = getSetColumnTypeData(); - if (!data || !data.columns) { - return null; - } - return data.columns[column] || null; -} - -function canSetColumnType() { - return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch); -} - -function setColumnTypeActionLabel(column) { - var columnConfig = getSetColumnTypeConfig(column); - if (!columnConfig) { - return null; - } - return columnConfig.current - ? `Custom type: ${columnConfig.current.type}` - : "Set custom type"; -} - -function createSetColumnTypeOption(value, name, description, checked) { - var label = document.createElement("label"); - label.className = "set-column-type-option"; - - var input = document.createElement("input"); - input.type = "radio"; - input.name = "set-column-type-choice"; - input.value = value; - input.checked = checked; - - var content = document.createElement("span"); - content.className = "set-column-type-option-content"; - - var title = document.createElement("span"); - title.className = "set-column-type-option-name"; - title.textContent = name; - - var detail = document.createElement("span"); - detail.className = "set-column-type-option-description"; - detail.textContent = description; - - content.appendChild(title); - content.appendChild(detail); - label.appendChild(input); - label.appendChild(content); - return label; -} - -function setSetColumnTypeDialogBusy(state, isBusy) { - state.isBusy = isBusy; - state.saveButton.disabled = isBusy; - state.cancelButton.disabled = isBusy; - Array.from( - state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'), - ).forEach(function (input) { - input.disabled = isBusy; - }); - state.saveButton.textContent = isBusy ? "Saving..." : "Save"; -} - -function clearSetColumnTypeDialogError(state) { - state.error.hidden = true; - state.error.textContent = ""; -} - -function showSetColumnTypeDialogError(state, message) { - state.error.hidden = false; - state.error.textContent = message; -} - -function ensureSetColumnTypeDialog() { - if (setColumnTypeDialogState) { - return setColumnTypeDialogState; - } - if (!window.HTMLDialogElement) { - return null; - } - - var dialog = document.createElement("dialog"); - dialog.id = SET_COLUMN_TYPE_DIALOG_ID; - dialog.className = "set-column-type-dialog"; - dialog.setAttribute("aria-labelledby", "set-column-type-title"); - dialog.innerHTML = ` -
- Set custom type - -
-

- -
-
- - - -
- `; - document.body.appendChild(dialog); - - setColumnTypeDialogState = { - dialog: dialog, - meta: dialog.querySelector(".modal-meta"), - status: dialog.querySelector(".set-column-type-status"), - error: dialog.querySelector(".set-column-type-error"), - optionsWrap: dialog.querySelector(".set-column-type-options"), - footerInfo: dialog.querySelector(".footer-info"), - cancelButton: dialog.querySelector(".set-column-type-cancel"), - saveButton: dialog.querySelector(".set-column-type-save"), - currentColumn: null, - currentConfig: null, - isBusy: false, - }; - - setColumnTypeDialogState.cancelButton.addEventListener("click", function () { - if (!setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("click", function (ev) { - if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { - dialog.close(); - } - }); - - dialog.addEventListener("cancel", function (ev) { - if (setColumnTypeDialogState.isBusy) { - ev.preventDefault(); - } - }); - - dialog.addEventListener("close", function () { - clearSetColumnTypeDialogError(setColumnTypeDialogState); - setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); - }); - - setColumnTypeDialogState.saveButton.addEventListener("click", async function () { - var state = setColumnTypeDialogState; - var selected = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - var selectedType = selected ? selected.value : ""; - var currentType = state.currentConfig.current - ? state.currentConfig.current.type - : ""; - - if (selectedType === currentType) { - state.dialog.close(); - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, true); - - var payload = { - column: state.currentColumn, - column_type: selectedType ? { type: selectedType } : null, - }; - - try { - var response = await fetch(getSetColumnTypeData().path, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(payload), - }); - var data = await response.json(); - if (!response.ok || data.ok === false) { - var message = (data.errors || ["Request failed"]).join(" "); - throw new Error(message); - } - location.reload(); - } catch (error) { - setSetColumnTypeDialogBusy(state, false); - showSetColumnTypeDialogError(state, error.message || "Request failed"); - } - }); - - return setColumnTypeDialogState; -} - -function openSetColumnTypeDialog(th) { - var column = th.dataset.column; - var columnConfig = getSetColumnTypeConfig(column); - if (!columnConfig) { - return; - } - - var state = ensureSetColumnTypeDialog(); - if (!state) { - return; - } - - clearSetColumnTypeDialogError(state); - setSetColumnTypeDialogBusy(state, false); - state.currentColumn = column; - state.currentConfig = columnConfig; - state.status.textContent = `Column: ${column}`; - state.meta.textContent = getColumnTypeText(th) || "Type unavailable"; - state.footerInfo.textContent = columnConfig.current - ? `Current custom type: ${columnConfig.current.type}` - : "No custom type set."; - state.optionsWrap.innerHTML = ""; - - var currentType = columnConfig.current ? columnConfig.current.type : ""; - state.optionsWrap.appendChild( - createSetColumnTypeOption( - "", - "No custom type", - "Use standard Datasette rendering without a custom type.", - currentType === "", - ), - ); - - columnConfig.options.forEach(function (option) { - state.optionsWrap.appendChild( - createSetColumnTypeOption( - option.name, - option.name, - option.description, - option.name === currentType, - ), - ); - }); - - if (!columnConfig.options.length) { - var emptyState = document.createElement("p"); - emptyState.className = "set-column-type-empty"; - emptyState.textContent = - "No registered custom types are compatible with this SQLite type."; - state.optionsWrap.appendChild(emptyState); - } - - if (!state.dialog.open) { - state.dialog.showModal(); - } - var selectedOption = state.dialog.querySelector( - 'input[name="set-column-type-choice"]:checked', - ); - if (selectedOption) { - selectedOption.focus(); - } else { - state.saveButton.focus(); - } -} - function canChooseColumns() { return !!( document.querySelector("column-chooser") && window._columnChooserData @@ -427,21 +171,6 @@ function buildColumnActionItems(manager, th, options) { }); } - if (canSetColumnType() && getSetColumnTypeConfig(column)) { - columnActions.push({ - label: setColumnTypeActionLabel(column), - href: "#", - onClick: - options.onSetColumnType || - function (ev) { - ev.preventDefault(); - window.setTimeout(function () { - openSetColumnTypeDialog(th); - }, 0); - }, - }); - } - if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) { columnActions.push({ label: "Hide this column", @@ -552,13 +281,6 @@ const initDatasetteTable = function (manager) { closeMenu(); openColumnChooser(); }, - onSetColumnType: function (ev) { - ev.preventDefault(); - closeMenu(); - window.setTimeout(function () { - openSetColumnTypeDialog(th); - }, 0); - }, }); var menuList = menu.querySelector("ul.dropdown-actions"); menuList.innerHTML = ""; diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py deleted file mode 100644 index bcfdfdb4..00000000 --- a/datasette/stored_queries.py +++ /dev/null @@ -1,623 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -from typing import Any, Iterable - -from .resources import TableResource -from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components -from .utils.asgi import Forbidden - -UNCHANGED = object() - -QUERY_OPTION_FIELDS = ( - "hide_sql", - "fragment", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", -) - - -@dataclass -class StoredQuery: - database: str - name: str - sql: str - title: str | None - description: str | None - description_html: str | None - hide_sql: bool - fragment: str | None - parameters: list[str] - is_write: bool - is_private: bool - is_trusted: bool - source: str - owner_id: str | None - on_success_message: str | None - on_success_message_sql: str | None - on_success_redirect: str | None - on_error_message: str | None - on_error_redirect: str | None - private: bool | None = None - - -@dataclass -class StoredQueryPage: - queries: list[StoredQuery] - next: str | None - has_more: bool - limit: int - - -def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: - data = { - "database": query.database, - "name": query.name, - "sql": query.sql, - "title": query.title, - "description": query.description, - "description_html": query.description_html, - "hide_sql": query.hide_sql, - "fragment": query.fragment, - "params": list(query.parameters), - "parameters": list(query.parameters), - "is_write": query.is_write, - "is_private": query.is_private, - "is_trusted": query.is_trusted, - "source": query.source, - "owner_id": query.owner_id, - "on_success_message": query.on_success_message, - "on_success_message_sql": query.on_success_message_sql, - "on_success_redirect": query.on_success_redirect, - "on_error_message": query.on_error_message, - "on_error_redirect": query.on_error_redirect, - } - if query.private is not None: - data["private"] = query.private - return data - - -def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: - return { - "queries": [stored_query_to_dict(query) for query in page.queries], - "next": page.next, - "has_more": page.has_more, - "limit": page.limit, - } - - -async def save_queries_from_config(datasette: Any) -> None: - # Apply configured query entries from datasette.yaml to the internal table. - await datasette.get_internal_database().execute_write( - "DELETE FROM queries WHERE source = 'config'" - ) - for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if not isinstance(query_config, dict): - query_config = {"sql": query_config} - await datasette.add_query( - dbname, - query_name, - query_config["sql"], - title=query_config.get("title"), - description=query_config.get("description"), - description_html=query_config.get("description_html"), - hide_sql=bool(query_config.get("hide_sql")), - fragment=query_config.get("fragment"), - parameters=query_config.get("params"), - is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), - is_trusted=bool(query_config.get("is_trusted", True)), - source="config", - on_success_message=query_config.get("on_success_message"), - on_success_message_sql=query_config.get("on_success_message_sql"), - on_success_redirect=query_config.get("on_success_redirect"), - on_error_message=query_config.get("on_error_message"), - on_error_redirect=query_config.get("on_error_redirect"), - ) - - -def query_row_to_stored_query( - row: Any, private: bool | None = None -) -> StoredQuery | None: - if row is None: - return None - parameters = json.loads(row["parameters"] or "[]") - options = json.loads(row["options"] or "{}") - return StoredQuery( - database=row["database_name"], - name=row["name"], - sql=row["sql"], - title=row["title"], - description=row["description"], - description_html=row["description_html"], - hide_sql=bool(options.get("hide_sql")), - fragment=options.get("fragment"), - parameters=parameters, - is_write=bool(row["is_write"]), - is_private=bool(row["is_private"]), - is_trusted=bool(row["is_trusted"]), - source=row["source"], - owner_id=row["owner_id"], - on_success_message=options.get("on_success_message"), - on_success_message_sql=options.get("on_success_message_sql"), - on_success_redirect=options.get("on_success_redirect"), - on_error_message=options.get("on_error_message"), - on_error_redirect=options.get("on_error_redirect"), - private=private, - ) - - -def query_options_json(options: dict[str, Any]) -> str: - options_dict = {} - for field in QUERY_OPTION_FIELDS: - value = options.get(field) - if field == "hide_sql": - if value: - options_dict[field] = True - elif value is not None: - options_dict[field] = value - return json.dumps(options_dict, sort_keys=True) - - -async def add_query( - datasette: Any, - database: str, - name: str, - sql: str, - *, - title: str | None = None, - description: str | None = None, - description_html: str | None = None, - hide_sql: bool = False, - fragment: str | None = None, - parameters: Iterable[str] | None = None, - is_write: bool = False, - is_private: bool = False, - is_trusted: bool = False, - source: str = "plugin", - owner_id: str | None = None, - on_success_message: str | None = None, - on_success_message_sql: str | None = None, - on_success_redirect: str | None = None, - on_error_message: str | None = None, - on_error_redirect: str | None = None, - replace: bool = True, -) -> None: - parameters_json = json.dumps(list(parameters or [])) - options_json = query_options_json( - { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - ) - sql_statement = """ - INSERT INTO queries ( - database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_private, is_trusted, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - if replace: - sql_statement += """ - ON CONFLICT(database_name, name) DO UPDATE SET - sql = excluded.sql, - title = excluded.title, - description = excluded.description, - description_html = excluded.description_html, - options = excluded.options, - parameters = excluded.parameters, - is_write = excluded.is_write, - is_private = excluded.is_private, - is_trusted = excluded.is_trusted, - source = excluded.source, - owner_id = excluded.owner_id, - updated_at = CURRENT_TIMESTAMP - """ - await datasette.get_internal_database().execute_write( - sql_statement, - [ - database, - name, - sql, - title, - description, - description_html, - options_json, - parameters_json, - int(bool(is_write)), - int(bool(is_private)), - int(bool(is_trusted)), - source, - owner_id, - ], - ) - - -async def update_query( - datasette: Any, - database: str, - name: str, - *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, -) -> None: - fields = { - "sql": sql, - "title": title, - "description": description, - "description_html": description_html, - "parameters": parameters, - "is_write": is_write, - "is_private": is_private, - "is_trusted": is_trusted, - "source": source, - "owner_id": owner_id, - } - option_fields = { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - updates = [] - params = [] - for field, value in fields.items(): - if value is UNCHANGED: - continue - if field in {"is_write", "is_private", "is_trusted"}: - value = int(bool(value)) - elif field == "parameters": - value = json.dumps(list(value or [])) - updates.append(f"{field} = ?") - params.append(value) - changed_options = { - field: value for field, value in option_fields.items() if value is not UNCHANGED - } - if changed_options: - rows = await datasette.get_internal_database().execute( - """ - SELECT options FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - row = rows.first() - options = json.loads(row["options"] or "{}") if row is not None else {} - for field, value in changed_options.items(): - if field == "hide_sql": - if value: - options[field] = True - else: - options.pop(field, None) - elif value is None: - options.pop(field, None) - else: - options[field] = value - updates.append("options = ?") - params.append(json.dumps(options, sort_keys=True)) - if not updates: - return - updates.append("updated_at = CURRENT_TIMESTAMP") - params.extend([database, name]) - await datasette.get_internal_database().execute_write( - """ - UPDATE queries - SET {} - WHERE database_name = ? AND name = ? - """.format(", ".join(updates)), - params, - ) - - -async def remove_query( - datasette: Any, database: str, name: str, source: str | None = None -) -> None: - sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" - params = [database, name] - if source is not None: - sql += " AND source = ?" - params.append(source) - await datasette.get_internal_database().execute_write(sql, params) - - -async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: - rows = await datasette.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - return query_row_to_stored_query(rows.first()) - - -async def count_queries( - datasette: Any, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, -) -> int: - allowed_sql, allowed_params = await datasette.allowed_resources_sql( - action="view-query", - actor=actor, - parent=database, - ) - params = dict(allowed_params) - where_clauses = [] - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - row = ( - await datasette.get_internal_database().execute( - """ - SELECT count(*) AS count - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - """.format( - allowed_sql=allowed_sql, - where=" AND ".join(where_clauses) or "1 = 1", - ), - params, - ) - ).first() - return row["count"] - - -async def list_queries( - datasette: Any, - database: str | None = None, - *, - actor: dict[str, Any] | None = None, - limit: int = 50, - cursor: str | None = None, - q: str | None = None, - is_write: bool | None = None, - is_private: bool | None = None, - is_trusted: bool | None = None, - source: str | None = None, - owner_id: str | None = None, - include_private: bool = False, -) -> StoredQueryPage: - limit = min(max(1, int(limit)), 1000) - allowed_sql, allowed_params = await datasette.allowed_resources_sql( - action="view-query", - actor=actor, - parent=database, - include_is_private=include_private, - ) - params = dict(allowed_params) - params.update({"limit": limit + 1}) - sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = [] - order_by = "q.database_name, sort_key, q.name" - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - order_by = "sort_key, q.name" - - if cursor: - try: - components = urlsafe_components(cursor) - except ValueError: - components = [] - if database is None and len(components) == 3: - where_clauses.append(""" - ( - q.database_name > :cursor_database - OR ( - q.database_name = :cursor_database - AND ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_database"] = components[0] - params["cursor_sort_key"] = components[1] - params["cursor_name"] = components[2] - elif database is not None and len(components) == 2: - where_clauses.append(""" - ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_sort_key"] = components[0] - params["cursor_name"] = components[1] - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - private_select = ", allowed.is_private AS private" if include_private else "" - rows = list( - ( - await datasette.get_internal_database().execute( - """ - SELECT q.*, {sort_key_sql} AS sort_key{private_select} - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - ORDER BY {order_by} - LIMIT :limit - """.format( - allowed_sql=allowed_sql, - private_select=private_select, - sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses) or "1 = 1", - order_by=order_by, - ), - params, - ) - ).rows - ) - has_more = len(rows) > limit - if has_more: - rows = rows[:limit] - - queries = [] - for row in rows: - query = query_row_to_stored_query( - row, private=bool(row["private"]) if include_private else None - ) - assert query is not None - queries.append(query) - - next_token = None - if has_more and rows: - last_row = rows[-1] - if database is None: - next_token = "{},{},{}".format( - tilde_encode(last_row["database_name"]), - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - else: - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - return StoredQueryPage( - queries=queries, - next=next_token, - has_more=has_more, - limit=limit, - ) - - -async def ensure_query_write_permissions( - datasette: Any, - database: str, - sql: str, - *, - actor: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - analysis: Any = None, -) -> Any: - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - db = datasette.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: - continue - if access.database != database: - raise Forbidden("Writable queries may not write to attached databases") - if not await datasette.allowed( - action=action, - resource=TableResource(database=access.database, table=access.table), - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) - return analysis diff --git a/datasette/templates/_action_menu.html b/datasette/templates/_action_menu.html index 1ae8c173..7d1d4a55 100644 --- a/datasette/templates/_action_menu.html +++ b/datasette/templates/_action_menu.html @@ -1,7 +1,7 @@ {% if action_links %}
-
@@ -31,7 +31,6 @@
-{% endif %} -{% if not display_rows %} +{% else %}

0 records

{% endif %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index e1767deb..0d89e11c 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -20,7 +20,7 @@