diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 18c01fdc..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000..5275ddef --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,48 @@ +name: Playwright + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + browser: [chromium, firefox, webkit] + steps: + - uses: actions/checkout@v6 + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: "3.14" + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml + - name: Cache uv + uses: actions/cache@v5 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py3.14-uv- + - name: Cache Playwright browsers + uses: actions/cache@v5 + with: + path: ~/.cache/ms-playwright/ + key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-playwright-${{ matrix.browser }}- + - name: Install uv + run: python -m pip install uv + - name: Install dependencies + run: uv sync --group dev --group playwright + - name: Install ${{ matrix.browser }} + run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }} + - name: Run Playwright tests + run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1b2e9d2..9e47db6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: diff --git a/.gitignore b/.gitignore index 12acd87e..2a7f6620 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ build-metadata.json datasets.json +.playwright-mcp + scratchpad +ignored/ + .vscode uv.lock @@ -131,4 +135,4 @@ tests/*.dylib tests/*.so tests/*.dll -.idea \ No newline at end of file +.idea diff --git a/Justfile b/Justfile index 657881be..5fcd9afd 100644 --- a/Justfile +++ b/Justfile @@ -11,6 +11,22 @@ export DATASETTE_SECRET := "not_a_secret" @test *options: init uv run pytest -n auto {{options}} +# Install Playwright browser support, Chromium by default +@playwright-install browser="chromium": + uv run --group playwright playwright install {{browser}} + +# Install all Playwright browsers used by the test suite +@playwright-install-all: + uv run --group playwright playwright install chromium firefox webkit + +# Run Playwright tests, Chromium by default +@playwright browser="chromium" *options: + uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}} + +# Run Playwright tests against all supported browsers +@playwright-all *options: + uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}} + @codespell: uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 5fb6b473..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -19,23 +19,38 @@ import weakref import pytest -from datasette.app import Datasette - _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( "datasette_active_instances", default=None ) -_original_init = Datasette.__init__ +_original_init = None -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init -Datasette.__init__ = _tracking_init +def pytest_configure(config): + if _enabled(config): + _install_tracking() def pytest_addoption(parser): diff --git a/datasette/app.py b/datasette/app.py index 75f05d88..9c9b7de4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence if TYPE_CHECKING: from datasette.permissions import Resource @@ -12,7 +12,6 @@ import dataclasses import datetime import functools import glob -import hashlib import httpx import importlib.metadata import inspect @@ -35,6 +34,7 @@ from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, + pass_context, PrefixLoader, ) from jinja2.environment import Template @@ -42,12 +42,36 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType +from . import stored_queries, write_sql from .views import Context -from .views.database import database_download, DatabaseView, TableCreateView, QueryView +from .views.database import ( + database_download, + DatabaseView, + QueryView, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + QueryEditView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) from .views.index import IndexView from .views.special import ( JsonDataView, PatternPortfolioView, + AutocompleteDebugView, AuthTokenView, ApiExplorerView, CreateTokenView, @@ -64,10 +88,12 @@ from .views.special import ( TableSchemaView, ) from .views.table import ( + TableAutocompleteView, TableInsertView, TableUpsertView, TableSetColumnTypeView, TableDropView, + TableFragmentView, table_view, ) from .views.row import RowView, RowDeleteView, RowUpdateView @@ -96,6 +122,7 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, + sha256_file, tilde_decode, tilde_encode, to_css_class, @@ -273,12 +300,21 @@ DEFAULT_NOT_SET = object() ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) +def _permission_cache_key(actor, action, parent, child): + # Key on the full serialized actor so actors differing in any field + # (e.g. token restrictions) never share cache entries + actor_key = ( + json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None + ) + return (actor_key, action, parent, child) + + async def favicon(request, send): await asgi_send_file( send, str(FAVICON_PATH), content_type="image/png", - headers={"Cache-Control": "max-age=3600, immutable, public"}, + headers={"Cache-Control": "max-age=3600, public"}, ) @@ -295,6 +331,57 @@ def _to_string(value): return json.dumps(value, default=str) +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + +def _resolve_static_asset_path(root_path, path): + root = Path(root_path).resolve() + full_path = (root / path).resolve() + try: + full_path.relative_to(root) + except ValueError: + raise ValueError("Static asset path cannot escape static root") from None + return full_path + + +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", + "show_logout": "True if the logout link should be shown in the navigation menu", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - + + diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html index 034e9678..570bb37e 100644 --- a/datasette/templates/_facet_results.html +++ b/datasette/templates/_facet_results.html @@ -12,9 +12,9 @@ + {% if queries_more %} +

View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

+ {% endif %} {% endif %} {% if tables %} @@ -64,7 +76,7 @@

{{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

-

{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

+

{% if table.count is none %}Many rows{% elif table.count_truncated %}>{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

{% endif %} {% endfor %} @@ -87,5 +99,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index add3154a..4f8106b8 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,7 +3,7 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} - + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} diff --git a/datasette/templates/debug_autocomplete.html b/datasette/templates/debug_autocomplete.html new file mode 100644 index 00000000..380639a3 --- /dev/null +++ b/datasette/templates/debug_autocomplete.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} + +{% block title %}Debug autocomplete{% endblock %} + +{% block extra_head %} +{{ super() }} + +{% endblock %} + +{% block content %} +

Debug autocomplete

+ + +

+ + +

+

+ + +

+

+ + +{% if error %} +

{{ error }}

+{% elif autocomplete_url %} +

{{ database_name }} / {{ table_name }}

+ {% if label_column %} +

Label column: {{ label_column }}

+ {% else %} +

No label column detected. Results will use primary key values.

+ {% endif %} +
+ + + + +
+

Selected row

+ 
No row selected.
+ +{% else %} +

Suggested tables

+ {% if suggestions %} +

Showing up to five tables with a detected label column.

+ + + + + + + + + + {% for suggestion in suggestions %} + + + + + + {% endfor %} + +
DatabaseTableLabel column
{{ suggestion.database }}{{ suggestion.table }}{{ suggestion.label_column }}
+ {% else %} +

No tables with detected label columns found.

+ {% endif %} +

Scanned {{ scanned }} table{% if scanned != 1 %}s{% endif %}{% if reached_scan_limit %}; stopped at the 100 table scan limit{% endif %}.

+{% endif %} + +{% endblock %} diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index c2e7997f..3b229a25 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,7 +3,7 @@ {% block title %}Permission Check{% endblock %} {% block extra_head %} - + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} +{% include "_execute_write_analysis_styles.html" %} +{% include "_sql_parameter_styles.html" %} +{% endblock %} + +{% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +

Write to this database

+ +

Execute SQL to insert, update or delete rows in this database.

+ +{% if execution_message %} +

{{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

+{% endif %} + +{% if execute_write_returns_rows %} +

Returned rows

+ {% if execute_write_truncated %} +

Only the first {{ "{:,}".format(execute_write_display_rows|length) }} returned rows are shown.

+ {% endif %} + {% set columns = execute_write_columns %} + {% set display_rows = execute_write_display_rows %} + {% set show_zero_results = true %} + {% include "_query_results.html" %} +{% endif %} + +
+ {% if write_create_table_template_sql or write_template_tables %} +
+
+ Start with a template +

+ {% if write_create_table_template_sql %} + + {% endif %} + {% if write_template_tables %} + + + {% for operation in write_template_operations %} + + {% endfor %} + {% endif %} +

+
+
+ {% else %} +

There are no tables that you can currently edit.

+ {% endif %} + +

+ + {% set sql_parameters_section_id = "execute-write-parameters-section" %} + {% set sql_parameters_allow_expand = true %} + {% include "_sql_parameters.html" %} + +
+

Query operations

+ {% if analysis_error %} +

{{ analysis_error }}

+ {% elif analysis_rows %} +
+ + + + + + + + + + + {% for row in analysis_rows %} + + + + + + + + {% endfor %} + +
OperationDatabaseTableRequired permissionAllowed
{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
+ {% else %} +

Analysis will show each affected table and required permission.

+ {% endif %} +
+ +

+ + {{ execute_disabled_reason or "" }} + {% if save_query_url %}Save this query{% endif %} +

+
+ +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + +{% if write_create_table_template_sql or write_template_tables %} + +{% endif %} + +{% endblock %} diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 7770f7d4..075c0117 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -2,7 +2,7 @@ Datasette: Pattern Portfolio - + @@ -11,7 +11,7 @@