From 435ff7fa88fce25f5f2e4fa0aae8834deff7240c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:51:09 -0700 Subject: [PATCH 01/20] Context.documented_fields() and extras doc-metadata enforcement - Context dataclasses now expose documented_fields(), returning ContextField(name, type_name, help) for each field - ExtraRegistry.internal_classes_for_scope() returns the Extra classes that are available to HTML templates but excluded from JSON - Tests enforce that every registered Extra has a description and every DatabaseContext/QueryContext field has help metadata Refs #1510, #2127 Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 10 ++++++++++ datasette/views/__init__.py | 25 +++++++++++++++++++++++++ tests/test_extras.py | 32 ++++++++++++++++++++++++++++++++ tests/test_template_context.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 tests/test_template_context.py diff --git a/datasette/extras.py b/datasette/extras.py index 5cab52a4..36014185 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -81,6 +81,16 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def internal_classes_for_scope(self, scope): + # Extras that are available to HTML templates but excluded from + # JSON responses - plain Providers are dependency plumbing and + # never surface as keys, so they are not included + return [ + cls + for cls in self.classes_for_scope(scope) + if issubclass(cls, Extra) and not cls.public + ] + def _registry_for_scope(self, scope): registry = self._scope_registries.get(scope) if registry is None: diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 88106737..de851708 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,2 +1,27 @@ +from dataclasses import dataclass +import dataclasses + + +@dataclass(frozen=True) +class ContextField: + name: str + type_name: str + help: str + + class Context: "Base class for all documented contexts" + + @classmethod + def documented_fields(cls): + "List of ContextField describing the documented fields of this context" + documented = [] + for f in dataclasses.fields(cls): + documented.append( + ContextField( + name=f.name, + type_name=getattr(f.type, "__name__", str(f.type)), + help=f.metadata.get("help", ""), + ) + ) + return documented diff --git a/tests/test_extras.py b/tests/test_extras.py index ad8a9f00..73b4965e 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -23,6 +23,38 @@ class DependentExtra(Extra): return slow_value + 1 +class InternalOnlyExtra(Extra): + description = "Internal extra for HTML templates only" + scopes = {ExtraScope.TABLE} + public = False + + async def resolve(self, context): + return "internal" + + +def test_internal_classes_for_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra, InternalOnlyExtra]) + assert registry.internal_classes_for_scope(ExtraScope.TABLE) == [InternalOnlyExtra] + assert registry.public_classes_for_scope(ExtraScope.TABLE) == [ + SlowValueExtra, + DependentExtra, + ] + + +def _registered_extra_classes(): + # Plain Providers are internal dependency plumbing, only Extra + # subclasses surface as documented JSON/template keys + from datasette.views.table_extras import table_extra_registry + + return [cls for cls in table_extra_registry.classes if issubclass(cls, Extra)] + + +@pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key()) +def test_registered_extras_have_descriptions(cls): + # Every registered extra is part of the documented template/JSON contract + assert cls.description, "{} is missing a description".format(cls.__name__) + + def test_registry_is_built_once_per_scope(): registry = ExtraRegistry([SlowValueExtra, DependentExtra]) first = registry._registry_for_scope(ExtraScope.TABLE) diff --git a/tests/test_template_context.py b/tests/test_template_context.py new file mode 100644 index 00000000..4ce00a55 --- /dev/null +++ b/tests/test_template_context.py @@ -0,0 +1,32 @@ +""" +Tests for the documented template context - the contract that custom +template authors can rely on for Datasette 1.0. +""" + +from dataclasses import dataclass, field + +import pytest + +from datasette.views import Context +from datasette.views.database import DatabaseContext, QueryContext + + +def test_documented_fields(): + @dataclass + class DemoContext(Context): + name: str = field(metadata={"help": "The name"}) + count: int = field(metadata={"help": "How many there are"}) + + fields = DemoContext.documented_fields() + assert [(f.name, f.type_name, f.help) for f in fields] == [ + ("name", "str", "The name"), + ("count", "int", "How many there are"), + ] + + +@pytest.mark.parametrize("klass", (DatabaseContext, QueryContext)) +def test_context_dataclass_fields_all_have_help(klass): + for context_field in klass.documented_fields(): + assert context_field.help, "{}.{} is missing help metadata".format( + klass.__name__, context_field.name + ) From 6a1b237b39ca11ca8a876f912b23e27f281eb0f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:55:25 -0700 Subject: [PATCH 02/20] Documented template context manifest with contract tests datasette/template_contexts.py is the source of truth for the template context contract: the variables custom templates can rely on for the database, query, table and row pages, plus the base context that render_template() adds to every page. Documentation for each key comes from the Context dataclass field help (database, query), the Extra class description (table and row extras) or inline docs in the manifest (keys added by view code). Contract tests render each page with template_debug ?_context=1 and assert the real context keys exactly match the documented set, in both directions - an undocumented addition or a removed documented key both fail. Refs #1510, #2127 Co-Authored-By: Claude Fable 5 --- datasette/template_contexts.py | 240 +++++++++++++++++++++++++++++++++ tests/test_template_context.py | 103 ++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 datasette/template_contexts.py diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py new file mode 100644 index 00000000..1ff1e2a8 --- /dev/null +++ b/datasette/template_contexts.py @@ -0,0 +1,240 @@ +""" +The documented template context for Datasette's core HTML pages. + +This module is the source of truth for the template context contract: +the set of variables that custom templates can rely on for each page. +It is consumed by the contract tests in tests/test_template_context.py, +which assert that the real rendered context for each page exactly +matches what is documented here, and by docs/template_context_doc.py +which generates the documentation in docs/template_context.rst. + +Documentation for each key comes from one of three places: + +- Pages that render a Context dataclass (database, query) use the + ``help`` metadata on each dataclass field +- Keys provided by registered extras use the ``description`` from the + Extra class +- Keys added inline by view code are documented in this module +""" + +from dataclasses import dataclass + +from datasette.extras import ExtraScope +from datasette.views.database import DatabaseContext, QueryContext +from datasette.views.table_extras import table_extra_registry + + +@dataclass(frozen=True) +class TemplateContextKey: + name: str + doc: str + + +def _keys(**docs): + return tuple(TemplateContextKey(name, doc) for name, doc in docs.items()) + + +# Added by Datasette.render_template() to the context for every page, +# including pages rendered by plugins +BASE_CONTEXT_KEYS = _keys( + request="The current Request object, or None", + crumb_items="Async function returning breadcrumb navigation items for the current page", + urls="Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", + actor="The currently authenticated actor dictionary, or None", + menu_links="Async function returning links for the Datasette application menu, including those added by plugins", + display_actor="Function returning a display string for an actor dictionary", + show_logout="True if the logout link should be shown in the navigation menu", + app_css_hash="Hash of Datasette's app.css contents, used for cache busting", + zip="Python's zip() builtin, made available to template logic", + body_scripts="List of script blocks for the page body contributed by plugins", + format_bytes="Function that formats a number of bytes as a human-readable size", + show_messages="Function returning any messages set for the current user, clearing them in the process", + extra_css_urls="List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", + extra_js_urls="List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", + base_url="The configured base_url setting", + csrftoken="Function returning the CSRF token for the current request", + datasette_version="The version of Datasette that is running", +) + + +@dataclass(frozen=True) +class PageContext: + # Identifier used in tests and documentation, e.g. "table" + name: str + title: str + description: str + # The default template used to render this page + template: str + # For pages rendered from a Context dataclass + context_class: type = None + # For pages whose context includes resolved extras + extras_scope: ExtraScope = None + extra_keys: tuple = () + # Keys added inline by the view code, documented here + keys: tuple = () + + def documented_keys(self): + "Every page-specific documented key, excluding BASE_CONTEXT_KEYS" + documented = [] + if self.context_class is not None: + documented.extend( + TemplateContextKey(f.name, f.help) + for f in self.context_class.documented_fields() + ) + for name in self.extra_keys: + cls = table_extra_registry.classes_by_name[name] + documented.append(TemplateContextKey(name, cls.description or "")) + documented.extend(self.keys) + return sorted(documented, key=lambda key: key.name) + + +_SHARED_DOCS = dict( + ok="True if the data for this page was retrieved without errors", + rows="The rows for this page, as a list of dictionaries mapping column name to value", + query_ms="Time taken by the SQL queries for this page, in milliseconds", + select_templates="List of template names that were considered for this page, the one used marked with an asterisk", + settings="Dictionary of Datasette's current settings", + alternate_url_json="URL for the JSON version of this page", + url_csv="URL for the CSV export of this page", + url_csv_path="Path portion of the CSV export URL", + url_csv_hidden_args="(name, value) pairs for hidden form fields used by the CSV export form", + renderers="Dictionary mapping output format names (e.g. json) to their URLs for this page", + display_columns="Column objects formatted for the HTML table display", + display_rows="Row data formatted for the HTML table display", + custom_table_templates="Custom template names that were considered for displaying this table", +) + + +def _shared(*names): + return tuple(TemplateContextKey(name, _SHARED_DOCS[name]) for name in names) + + +PAGES = { + page.name: page + for page in ( + PageContext( + name="database", + title="Database", + description="The page listing the tables, views and queries in a database, e.g. /fixtures", + template="database.html", + context_class=DatabaseContext, + ), + PageContext( + name="query", + title="Query", + description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)", + template="query.html", + context_class=QueryContext, + ), + PageContext( + name="table", + title="Table", + description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable", + template="table.html", + extras_scope=ExtraScope.TABLE, + extra_keys=( + "actions", + "all_columns", + "columns", + "count", + "count_sql", + "custom_table_templates", + "database", + "database_color", + "display_columns", + "display_rows", + "expandable_columns", + "facet_results", + "facets_timed_out", + "filters", + "form_hidden_args", + "human_description_en", + "is_view", + "metadata", + "next_url", + "primary_keys", + "private", + "query", + "renderers", + "set_column_type_ui", + "sorted_facet_results", + "suggested_facets", + "table", + "table_definition", + "view_definition", + ), + keys=_shared( + "ok", + "rows", + "query_ms", + "select_templates", + "settings", + "alternate_url_json", + "url_csv", + "url_csv_path", + "url_csv_hidden_args", + ) + + _keys( + allow_execute_sql="True if the current actor can execute custom SQL against this database", + append_querystring="Function that appends additional querystring arguments to a URL", + count_limit="The maximum number of rows Datasette will count before showing an approximation", + datasette_allow_facet='The string "true" or "false" reflecting the allow_facet setting', + extra_wheres_for_ui="Extra where clauses from ?_where=, with links to remove them", + filter_columns="List of columns offered by the filter interface", + fix_path="Function that applies the base_url prefix to a path", + is_sortable="True if any of the displayed columns can be used to sort", + next="Pagination token for the next page, or None", + path_with_replaced_args="Function for building the current path with modified querystring arguments", + sort="Column the page is sorted by, or None", + sort_desc="Column the page is sorted by in descending order, or None", + supports_search="True if this table has full-text search configured", + top_table="Async function rendering the top_table plugin slot", + ), + ), + PageContext( + name="row", + title="Row", + description="The page showing an individual row, e.g. /fixtures/facetable/1", + template="row.html", + extras_scope=ExtraScope.ROW, + extra_keys=( + "columns", + "database", + "database_color", + "foreign_key_tables", + "metadata", + "primary_keys", + "private", + "table", + ), + keys=_shared( + "ok", + "rows", + "query_ms", + "select_templates", + "settings", + "alternate_url_json", + "url_csv", + "url_csv_path", + "url_csv_hidden_args", + "renderers", + "display_columns", + "display_rows", + "custom_table_templates", + ) + + _keys( + primary_key_values="Values of the primary keys for this row, from the URL", + row_actions="Row actions made available by plugin hooks", + top_row="Async function rendering the top_row plugin slot", + ), + ), + ) +} + + +def documented_context_keys(page_name): + "Set of every documented key for the named page, including base context keys" + page = PAGES[page_name] + return {key.name for key in BASE_CONTEXT_KEYS} | { + key.name for key in page.documented_keys() + } diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4ce00a55..04002f31 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -3,12 +3,23 @@ Tests for the documented template context - the contract that custom template authors can rely on for Datasette 1.0. """ +import html +import json from dataclasses import dataclass, field import pytest +from datasette.app import Datasette +from datasette.extras import ExtraScope +from datasette.fixtures import write_fixture_database +from datasette.template_contexts import ( + BASE_CONTEXT_KEYS, + PAGES, + documented_context_keys, +) from datasette.views import Context from datasette.views.database import DatabaseContext, QueryContext +from datasette.views.table_extras import table_extra_registry def test_documented_fields(): @@ -30,3 +41,95 @@ def test_context_dataclass_fields_all_have_help(klass): assert context_field.help, "{}.{} is missing help metadata".format( klass.__name__, context_field.name ) + + +@pytest.fixture(scope="module") +def context_ds(tmp_path_factory): + db_path = tmp_path_factory.mktemp("template-context") / "fixtures.db" + write_fixture_database(db_path) + ds = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1, "template_debug": True}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": ( + "select _neighborhood from facetable " + "where _neighborhood like '%' || :text || '%'" + ), + "title": "Search neighborhoods", + } + } + } + } + }, + ) + yield ds + for db in ds.databases.values(): + if not db.is_memory: + db.close() + + +async def get_template_context(ds, path): + sep = "&" if "?" in path else "?" + response = await ds.client.get(path + sep + "_context=1") + assert response.status_code == 200, path + body = html.unescape( + response.text.removeprefix("
").removesuffix("
") + ) + return json.loads(body) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "page_name,path", + ( + ("database", "/fixtures"), + ("table", "/fixtures/facetable"), + ("table", "/fixtures/facetable?_city_id__exact=1"), + ("row", "/fixtures/facetable/1"), + ("query", "/fixtures/-/query?sql=select+*+from+facetable"), + ("query", "/fixtures/neighborhood_search?text=cork"), + ), +) +async def test_template_context_matches_documented_contract( + context_ds, page_name, path +): + # The full contract: every key in the rendered template context is + # documented, and every documented key is present in the context + documented = documented_context_keys(page_name) + actual = set(await get_template_context(context_ds, path)) + undocumented = actual - documented + no_longer_present = documented - actual + assert not undocumented, ( + "Undocumented keys in {} template context: {} - document them in " + "datasette/template_contexts.py".format(page_name, sorted(undocumented)) + ) + assert not no_longer_present, ( + "Documented keys missing from {} template context: {} - this would " + "break custom templates".format(page_name, sorted(no_longer_present)) + ) + + +def test_base_context_keys_all_have_docs(): + for key in BASE_CONTEXT_KEYS: + assert key.doc, "Base context key {} is missing docs".format(key.name) + + +@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) +def test_page_documented_keys_all_have_docs(page): + for key in page.documented_keys(): + assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) + + +@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) +def test_page_extra_keys_are_registered_extras(page): + for name in page.extra_keys: + cls = table_extra_registry.classes_by_name.get(name) + assert cls is not None, "{} is not a registered extra".format(name) + assert page.extras_scope is not None + assert cls.available_for(page.extras_scope), ( + "{} extra is not available for scope {}".format(name, page.extras_scope) + ) From a55ae2adfc9500dfbb028ac167c379d8207f4676 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 00:00:42 -0700 Subject: [PATCH 03/20] Generated template context documentation, closes #1510 docs/template_context.rst is generated by cog from the manifest in datasette/template_contexts.py, following the json_api_doc.py pattern. It documents the base context available on every page plus the database, query, table and row pages, including the stability policy for custom template authors. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/template_contexts.py | 8 +- docs/custom_templates.rst | 5 + docs/index.rst | 1 + docs/template_context.rst | 488 +++++++++++++++++++++++++++++++++ docs/template_context_doc.py | 54 ++++ tests/test_template_context.py | 25 +- 6 files changed, 571 insertions(+), 10 deletions(-) create mode 100644 docs/template_context.rst create mode 100644 docs/template_context_doc.py diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index 1ff1e2a8..e2854467 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -115,21 +115,21 @@ PAGES = { PageContext( name="database", title="Database", - description="The page listing the tables, views and queries in a database, e.g. /fixtures", + description="The page listing the tables, views and queries in a database, e.g. /fixtures.", template="database.html", context_class=DatabaseContext, ), PageContext( name="query", title="Query", - description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)", + description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name).", template="query.html", context_class=QueryContext, ), PageContext( name="table", title="Table", - description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable", + description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable.", template="table.html", extras_scope=ExtraScope.TABLE, extra_keys=( @@ -194,7 +194,7 @@ PAGES = { PageContext( name="row", title="Row", - description="The page showing an individual row, e.g. /fixtures/facetable/1", + description="The page showing an individual row, e.g. /fixtures/facetable/1.", template="row.html", extras_scope=ExtraScope.ROW, extra_keys=( diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index c324fb79..8066d28f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -177,6 +177,11 @@ this:: Datasette will now first look for templates in that directory, and fall back on the defaults if no matches are found. +The variables made available to each template are documented on the +:ref:`template_context` page. Variables documented there are a stable API: +custom templates that use them will keep working in future Datasette +releases, up until the next major version. + It is also possible to over-ride templates on a per-database, per-row or per- table basis. diff --git a/docs/index.rst b/docs/index.rst index c76969bc..d494fd17 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ Contents settings introspection custom_templates + template_context plugins writing_plugins javascript_plugins diff --git a/docs/template_context.rst b/docs/template_context.rst new file mode 100644 index 00000000..a487a6e7 --- /dev/null +++ b/docs/template_context.rst @@ -0,0 +1,488 @@ +.. _template_context: + +Template context +================ + +This page documents the variables that are available to custom templates +for each of Datasette's core pages. See :ref:`customization_custom_templates` +for how to provide your own templates. + +The variables documented here are a stable contract: custom templates that +use them will continue to work across Datasette releases, up until the next +major version (Datasette 2.0). Anything present in the template context but +not documented on this page is not part of that contract and may change or +be removed in any release. + +You can inspect the full context for any page by starting Datasette with +``--setting template_debug 1`` and adding ``?_context=1`` to the page URL. + +.. [[[cog + from template_context_doc import template_context + template_context(cog) +.. ]]] + +Base context +------------ + +These variables are available on every page rendered by Datasette, including pages rendered by plugins that use :ref:`datasette.render_template() `. Plugins can add additional variables using the :ref:`plugin_hook_extra_template_vars` hook. + +``request`` + The current Request object, or None + +``crumb_items`` + Async function returning breadcrumb navigation items for the current page + +``urls`` + Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation + +``actor`` + The currently authenticated actor dictionary, or None + +``menu_links`` + Async function returning links for the Datasette application menu, including those added by plugins + +``display_actor`` + Function returning a display string for an actor dictionary + +``show_logout`` + True if the logout link should be shown in the navigation menu + +``app_css_hash`` + Hash of Datasette's app.css contents, used for cache busting + +``zip`` + Python's zip() builtin, made available to template logic + +``body_scripts`` + List of script blocks for the page body contributed by plugins + +``format_bytes`` + Function that formats a number of bytes as a human-readable size + +``show_messages`` + Function returning any messages set for the current user, clearing them in the process + +``extra_css_urls`` + List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration + +``extra_js_urls`` + List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page + +``base_url`` + The configured base_url setting + +``csrftoken`` + Function returning the CSRF token for the current request + +``datasette_version`` + The version of Datasette that is running + +Database page +------------- + +The page listing the tables, views and queries in a database, e.g. /fixtures. Rendered using the ``database.html`` template. + +``allow_download`` - ``bool`` + Boolean indicating if database download is allowed + +``allow_execute_sql`` - ``bool`` + Boolean indicating if custom SQL can be executed + +``alternate_url_json`` - ``str`` + URL for the alternate JSON version of this page + +``attached_databases`` - ``list`` + List of names of attached databases + +``count_limit`` - ``int`` + The maximum number of rows to count + +``database`` - ``str`` + The name of the database + +``database_actions`` - ``callable`` + Callable returning list of action links for the database menu + +``database_color`` - ``str`` + The color assigned to the database + +``editable`` - ``bool`` + Boolean indicating if the database is editable + +``hidden_count`` - ``int`` + Count of hidden tables + +``metadata`` - ``dict`` + Metadata for the database + +``path`` - ``str`` + The URL path to this database + +``private`` - ``bool`` + Boolean indicating if this is a private database + +``queries`` - ``list`` + List of stored query objects + +``queries_count`` - ``int`` + Count of visible stored queries + +``queries_more`` - ``bool`` + Boolean indicating if more stored queries are available + +``select_templates`` - ``list`` + List of templates that were considered for rendering this page + +``show_hidden`` - ``str`` + Value of _show_hidden query parameter + +``size`` - ``int`` + The size of the database in bytes + +``table_columns`` - ``dict`` + Dictionary mapping table names to their column lists + +``tables`` - ``list`` + List of table objects in the database + +``top_database`` - ``callable`` + Callable to render the top_database slot + +``views`` - ``list`` + List of view objects in the database + +Query page +---------- + +The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name). Rendered using the ``query.html`` template. + +``allow_execute_sql`` - ``bool`` + Boolean indicating if custom SQL can be executed + +``alternate_url_json`` - ``str`` + URL for alternate JSON version of this page + +``columns`` - ``list`` + List of column names + +``database`` - ``str`` + The name of the database being queried + +``database_color`` - ``str`` + The color of the database + +``db_is_immutable`` - ``bool`` + Boolean indicating if this database is immutable + +``display_rows`` - ``list`` + List of result rows to display + +``edit_sql_url`` - ``str`` + URL to edit the SQL for a stored query + +``editable`` - ``bool`` + Boolean indicating if the SQL can be edited + +``error`` - ``str`` + Any query error message + +``hide_sql`` - ``bool`` + Boolean indicating if the SQL should be hidden + +``metadata`` - ``dict`` + Metadata about the database or the stored query + +``named_parameter_values`` - ``dict`` + Dictionary of parameter names/values + +``private`` - ``bool`` + Boolean indicating if this is a private database + +``query`` - ``dict`` + The SQL query object containing the `sql` string + +``query_actions`` - ``callable`` + Callable returning a list of links for the query action menu + +``renderers`` - ``dict`` + Dictionary of renderer name to URL + +``save_query_url`` - ``str`` + URL to save the current arbitrary SQL as a query + +``select_templates`` - ``list`` + List of templates that were considered for rendering this page + +``show_hide_hidden`` - ``str`` + Hidden input field for the _show_sql parameter + +``show_hide_link`` - ``str`` + The URL to toggle showing/hiding the SQL + +``show_hide_text`` - ``str`` + The text for the show/hide SQL link + +``stored_query`` - ``str`` + The name of the stored query if this is a stored query + +``stored_query_write`` - ``bool`` + Boolean indicating if this is a stored query that allows writes + +``table_columns`` - ``dict`` + Dictionary of table name to list of column names + +``tables`` - ``list`` + List of table objects in the database + +``top_query`` - ``callable`` + Callable to render the top_query slot + +``top_stored_query`` - ``callable`` + Callable to render the top_stored_query slot + +``url_csv`` - ``str`` + URL for CSV export + +Table page +---------- + +The page showing the rows in a table or SQL view, e.g. /fixtures/facetable. Rendered using the ``table.html`` template. + +Many of these keys are shared with the :ref:`JSON API ` for this page. + +``actions`` + Table or view actions made available by plugin hooks + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + +``allow_execute_sql`` + True if the current actor can execute custom SQL against this database + +``alternate_url_json`` + URL for the JSON version of this page + +``append_querystring`` + Function that appends additional querystring arguments to a URL + +``columns`` + Column names returned by this query + +``count`` + Total count of rows matching these filters + +``count_limit`` + The maximum number of rows Datasette will count before showing an approximation + +``count_sql`` + SQL query used to calculate the total count + +``custom_table_templates`` + Custom template names considered for this table + +``database`` + Database name + +``database_color`` + Color assigned to the database + +``datasette_allow_facet`` + The string "true" or "false" reflecting the allow_facet setting + +``display_columns`` + Column metadata used by the HTML table display + +``display_rows`` + Row data formatted for the HTML table display + +``expandable_columns`` + Foreign key columns that can be expanded with labels + +``extra_wheres_for_ui`` + Extra where clauses from ?_where=, with links to remove them + +``facet_results`` + Results of facets calculated against this data + +``facets_timed_out`` + Facet calculations that timed out + +``filter_columns`` + List of columns offered by the filter interface + +``filters`` + Filters object used by the HTML table interface + +``fix_path`` + Function that applies the base_url prefix to a path + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + +``human_description_en`` + Human-readable description of the filters + +``is_sortable`` + True if any of the displayed columns can be used to sort + +``is_view`` + Whether this resource is a view instead of a table + +``metadata`` + Metadata about the table, database or stored query + +``next`` + Pagination token for the next page, or None + +``next_url`` + Full URL for the next page of results + +``ok`` + True if the data for this page was retrieved without errors + +``path_with_replaced_args`` + Function for building the current path with modified querystring arguments + +``primary_keys`` + Primary keys for this table + +``private`` + Whether this resource is private to the current actor + +``query`` + Details of the underlying SQL query + +``query_ms`` + Time taken by the SQL queries for this page, in milliseconds + +``renderers`` + Alternative output renderers available for this table + +``rows`` + The rows for this page, as a list of dictionaries mapping column name to value + +``select_templates`` + List of template names that were considered for this page, the one used marked with an asterisk + +``set_column_type_ui`` + Column type UI metadata for this table + +``settings`` + Dictionary of Datasette's current settings + +``sort`` + Column the page is sorted by, or None + +``sort_desc`` + Column the page is sorted by in descending order, or None + +``sorted_facet_results`` + Facet results sorted for display + +``suggested_facets`` + Suggestions for facets that might return interesting results + +``supports_search`` + True if this table has full-text search configured + +``table`` + Table name + +``table_definition`` + SQL definition for this table + +``top_table`` + Async function rendering the top_table plugin slot + +``url_csv`` + URL for the CSV export of this page + +``url_csv_hidden_args`` + (name, value) pairs for hidden form fields used by the CSV export form + +``url_csv_path`` + Path portion of the CSV export URL + +``view_definition`` + SQL definition for this view + +Row page +-------- + +The page showing an individual row, e.g. /fixtures/facetable/1. Rendered using the ``row.html`` template. + +Many of these keys are shared with the :ref:`JSON API ` for this page. + +``alternate_url_json`` + URL for the JSON version of this page + +``columns`` + Column names returned by this query + +``custom_table_templates`` + Custom template names that were considered for displaying this table + +``database`` + Database name + +``database_color`` + Color assigned to the database + +``display_columns`` + Column objects formatted for the HTML table display + +``display_rows`` + Row data formatted for the HTML table display + +``foreign_key_tables`` + Tables that link to this row using foreign keys + +``metadata`` + Metadata about the table, database or stored query + +``ok`` + True if the data for this page was retrieved without errors + +``primary_key_values`` + Values of the primary keys for this row, from the URL + +``primary_keys`` + Primary keys for this table + +``private`` + Whether this resource is private to the current actor + +``query_ms`` + Time taken by the SQL queries for this page, in milliseconds + +``renderers`` + Dictionary mapping output format names (e.g. json) to their URLs for this page + +``row_actions`` + Row actions made available by plugin hooks + +``rows`` + The rows for this page, as a list of dictionaries mapping column name to value + +``select_templates`` + List of template names that were considered for this page, the one used marked with an asterisk + +``settings`` + Dictionary of Datasette's current settings + +``table`` + Table name + +``top_row`` + Async function rendering the top_row plugin slot + +``url_csv`` + URL for the CSV export of this page + +``url_csv_hidden_args`` + (name, value) pairs for hidden form fields used by the CSV export form + +``url_csv_path`` + Path portion of the CSV export URL + +.. [[[end]]] diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py new file mode 100644 index 00000000..1539dc5e --- /dev/null +++ b/docs/template_context_doc.py @@ -0,0 +1,54 @@ +""" +Cog helpers for generating docs/template_context.rst from the manifest +in datasette/template_contexts.py - same pattern as json_api_doc.py. +""" + + +def template_context(cog): + from datasette.template_contexts import BASE_CONTEXT_KEYS, PAGES + + cog.out("\n") + _section( + cog, + "Base context", + ( + "These variables are available on every page rendered by " + "Datasette, including pages rendered by plugins that use " + ":ref:`datasette.render_template() `. " + "Plugins can add additional variables using the " + ":ref:`plugin_hook_extra_template_vars` hook." + ), + ) + _untyped_keys(cog, BASE_CONTEXT_KEYS) + + for page in PAGES.values(): + _section( + cog, + "{} page".format(page.title), + "{} Rendered using the ``{}`` template.".format( + page.description, page.template + ), + ) + if page.context_class is not None: + for f in sorted( + page.context_class.documented_fields(), key=lambda f: f.name + ): + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) + else: + cog.out( + "Many of these keys are shared with the :ref:`JSON API " + "` for this page.\n\n" + ) + _untyped_keys(cog, page.documented_keys()) + + +def _section(cog, title, intro): + cog.out("{}\n{}\n\n".format(title, "-" * len(title))) + cog.out("{}\n\n".format(intro)) + + +def _untyped_keys(cog, keys): + for key in keys: + cog.out("``{}``\n".format(key.name)) + cog.out(" {}\n\n".format(key.doc)) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 04002f31..73e1d6c1 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -5,6 +5,7 @@ template authors can rely on for Datasette 1.0. import html import json +import pathlib from dataclasses import dataclass, field import pytest @@ -76,9 +77,7 @@ async def get_template_context(ds, path): sep = "&" if "?" in path else "?" response = await ds.client.get(path + sep + "_context=1") assert response.status_code == 200, path - body = html.unescape( - response.text.removeprefix("
").removesuffix("
") - ) + body = html.unescape(response.text.removeprefix("
").removesuffix("
")) return json.loads(body) @@ -124,12 +123,26 @@ def test_page_documented_keys_all_have_docs(page): assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) +def test_template_context_docs_cover_every_documented_key(): + docs_path = pathlib.Path(__file__).parent.parent / "docs" / "template_context.rst" + assert docs_path.exists(), "docs/template_context.rst is missing" + docs = docs_path.read_text() + for key in BASE_CONTEXT_KEYS: + assert "``{}``".format(key.name) in docs, key.name + for page in PAGES.values(): + assert page.title in docs, page.title + for key in page.documented_keys(): + assert "``{}``".format(key.name) in docs, "{} ({} page)".format( + key.name, page.name + ) + + @pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) def test_page_extra_keys_are_registered_extras(page): for name in page.extra_keys: cls = table_extra_registry.classes_by_name.get(name) assert cls is not None, "{} is not a registered extra".format(name) assert page.extras_scope is not None - assert cls.available_for(page.extras_scope), ( - "{} extra is not available for scope {}".format(name, page.extras_scope) - ) + assert cls.available_for( + page.extras_scope + ), "{} extra is not available for scope {}".format(name, page.extras_scope) From 3ea7ed86065d3071fa76c755bdfa63e6776ea7cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 00:06:56 -0700 Subject: [PATCH 04/20] Isolate test plugins from template context contract tests Datasette instances created with plugins_dir register their plugins on the global plugin manager for the rest of the process, so the contract tests could see extra_template_vars keys leaked from earlier test modules (e.g. the session-scoped ds_client fixture). A fixture now unregisters non-default plugins implementing extra_template_vars for the duration of each contract test and restores them afterwards. Co-Authored-By: Claude Fable 5 --- tests/test_template_context.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 73e1d6c1..3c6b13de 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -44,6 +44,27 @@ def test_context_dataclass_fields_all_have_help(klass): ) +@pytest.fixture +def isolate_extra_template_vars_plugins(): + # Datasette instances created with plugins_dir (e.g. the session-scoped + # ds_client fixture) register their plugins on the global plugin manager + # for the rest of the process. The contract documents plugin-free + # Datasette core, so unregister any non-default plugin that adds + # template variables via the extra_template_vars hook + from datasette.plugins import pm, DEFAULT_PLUGINS + + hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} + removed = [] + for plugin in list(pm.get_plugins()): + name = pm.get_name(plugin) + if name not in DEFAULT_PLUGINS and plugin in hook_plugins: + pm.unregister(plugin) + removed.append((plugin, name)) + yield + for plugin, name in removed: + pm.register(plugin, name) + + @pytest.fixture(scope="module") def context_ds(tmp_path_factory): db_path = tmp_path_factory.mktemp("template-context") / "fixtures.db" @@ -94,7 +115,7 @@ async def get_template_context(ds, path): ), ) async def test_template_context_matches_documented_contract( - context_ds, page_name, path + context_ds, isolate_extra_template_vars_plugins, page_name, path ): # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context From 63995ce823124569968e9dfdcc94221c59a83810 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:51:54 -0700 Subject: [PATCH 05/20] extra_field() - Context fields documented by their Extra class A Context dataclass field declared with extra_field() takes its documentation from the description on the registered Extra of the same name, validated against the class's extras_scope. This keeps doc strings next to the resolve() code instead of duplicating them on the dataclass, ahead of introducing TableContext and RowContext. Co-Authored-By: Claude Fable 5 --- datasette/views/__init__.py | 47 +++++++++++++++++++++++++++++++++- tests/test_template_context.py | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index de851708..238b9bb3 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -7,21 +7,66 @@ class ContextField: name: str type_name: str help: str + from_extra: bool = False + + +def extra_field(): + """ + Declare a Context dataclass field whose value comes from a registered + Extra of the same name - its documentation is the Extra description, + so the doc string lives next to the resolve() code rather than being + duplicated on the dataclass. + """ + return dataclasses.field(metadata={"from_extra": True}) class Context: "Base class for all documented contexts" + # Set on subclasses whose extra_field() fields should be resolved + # against the extras registry for this scope + extras_scope = None + @classmethod def documented_fields(cls): "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + from_extra = bool(f.metadata.get("from_extra")) + if from_extra: + help_text = cls._extra_description(f.name) + else: + help_text = f.metadata.get("help", "") documented.append( ContextField( name=f.name, type_name=getattr(f.type, "__name__", str(f.type)), - help=f.metadata.get("help", ""), + help=help_text, + from_extra=from_extra, ) ) return documented + + @classmethod + def _extra_description(cls, name): + # Imported lazily - table_extras is not needed just to define + # Context subclasses + from datasette.views.table_extras import table_extra_registry + + try: + extra_class = table_extra_registry.classes_by_name[name] + except KeyError: + raise KeyError( + "{}.{} is declared with extra_field() but there is no " + "registered extra of that name".format(cls.__name__, name) + ) + if cls.extras_scope is not None and not extra_class.available_for( + cls.extras_scope + ): + raise ValueError( + "{}.{} is declared with extra_field() but the {} extra is " + "not available for scope {}".format( + cls.__name__, name, name, cls.extras_scope + ) + ) + return extra_class.description or "" diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 3c6b13de..c385097a 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -44,6 +44,51 @@ def test_context_dataclass_fields_all_have_help(klass): ) +def test_extra_field_documentation_comes_from_the_extra_class(): + from datasette.views import extra_field + from datasette.views.table_extras import CountExtra + + @dataclass + class DemoContext(Context): + extras_scope = ExtraScope.TABLE + + count: int = extra_field() + name: str = field(metadata={"help": "The name"}) + + fields = {f.name: f for f in DemoContext.documented_fields()} + assert fields["count"].help == CountExtra.description + assert fields["count"].from_extra + assert fields["name"].help == "The name" + assert not fields["name"].from_extra + + +def test_extra_field_must_match_a_registered_extra(): + from datasette.views import extra_field + + @dataclass + class BadContext(Context): + extras_scope = ExtraScope.TABLE + + not_a_real_extra: str = extra_field() + + with pytest.raises(KeyError): + BadContext.documented_fields() + + +def test_extra_field_must_be_available_for_the_scope(): + from datasette.views import extra_field + + @dataclass + class WrongScopeContext(Context): + extras_scope = ExtraScope.ROW + + # count is a TABLE-scope extra, not available for ROW + count: int = extra_field() + + with pytest.raises(ValueError): + WrongScopeContext.documented_fields() + + @pytest.fixture def isolate_extra_template_vars_plugins(): # Datasette instances created with plugins_dir (e.g. the session-scoped From 8b89a3aca85bdf8eee93b9f4677dd85b2599dcb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:57:51 -0700 Subject: [PATCH 06/20] TableContext - table page now renders a documented Context dataclass The table HTML view constructs a TableContext instead of an ad-hoc dict, matching how the database and query pages already work. Fields resolved by registered extras are declared with extra_field() so their documentation lives on the Extra classes in table_extras.py; fields added by the view code carry help metadata next to the view. render_template() now converts Context dataclasses shallowly instead of via dataclasses.asdict(), which deep-copied every value and would fail on values like sqlite3.Row. Keys not declared on TableContext - extras requested with ?_extra= on the HTML page, or extra filter context from filters_from_request plugins - are now dropped from the HTML template context rather than passed through undocumented. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/app.py | 6 +- datasette/views/table.py | 136 +++++++++++++++++++++++++++++- tests/test_internals_datasette.py | 25 ++++++ tests/test_template_context.py | 8 ++ 4 files changed, 172 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 81d23acb..275baae4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2162,7 +2162,11 @@ class Datasette: templates = [templates] template = self.get_jinja_environment(request).select_template(templates) if dataclasses.is_dataclass(context): - context = dataclasses.asdict(context) + # Shallow conversion - asdict() would deep-copy values, which + # is wasteful and fails on values like sqlite3.Row + context = { + f.name: getattr(context, f.name) for f in dataclasses.fields(context) + } body_scripts = [] # pylint: disable=no-member for extra_script in pm.hook.extra_body_script( diff --git a/datasette/views/table.py b/datasette/views/table.py index 65388c9c..356247ff 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -43,6 +43,10 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters import sqlite_utils +from dataclasses import dataclass, field, fields + +from datasette.extras import ExtraScope +from . import Context, extra_field from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_extras import ( @@ -52,6 +56,129 @@ from .table_extras import ( table_extra_registry, ) + +@dataclass +class TableContext(Context): + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable" + + extras_scope = ExtraScope.TABLE + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + actions: callable = extra_field() + all_columns: list = extra_field() + columns: list = extra_field() + count: int = extra_field() + count_sql: str = extra_field() + custom_table_templates: list = extra_field() + database: str = extra_field() + database_color: str = extra_field() + display_columns: list = extra_field() + display_rows: list = extra_field() + expandable_columns: list = extra_field() + facet_results: dict = extra_field() + facets_timed_out: list = extra_field() + filters: Filters = extra_field() + form_hidden_args: list = extra_field() + human_description_en: str = extra_field() + is_view: bool = extra_field() + metadata: dict = extra_field() + next_url: str = extra_field() + primary_keys: list = extra_field() + private: bool = extra_field() + query: dict = extra_field() + renderers: dict = extra_field() + set_column_type_ui: dict = extra_field() + sorted_facet_results: list = extra_field() + suggested_facets: list = extra_field() + table: str = extra_field() + table_definition: str = extra_field() + view_definition: str = extra_field() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to value" + } + ) + filter_columns: list = field( + metadata={"help": "List of columns offered by the filter interface"} + ) + supports_search: bool = field( + metadata={"help": "True if this table has full-text search configured"} + ) + extra_wheres_for_ui: list = field( + metadata={ + "help": "Extra where clauses from ?_where=, with links to remove them" + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "(name, value) pairs for hidden form fields used by the CSV export form" + } + ) + sort: str = field(metadata={"help": "Column the page is sorted by, or None"}) + sort_desc: str = field( + metadata={"help": "Column the page is sorted by in descending order, or None"} + ) + append_querystring: callable = field( + metadata={ + "help": "Function that appends additional querystring arguments to a URL" + } + ) + path_with_replaced_args: callable = field( + metadata={ + "help": "Function for building the current path with modified querystring arguments" + } + ) + fix_path: callable = field( + metadata={"help": "Function that applies the base_url prefix to a path"} + ) + settings: dict = field( + metadata={"help": "Dictionary of Datasette's current settings"} + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + datasette_allow_facet: str = field( + metadata={ + "help": 'The string "true" or "false" reflecting the allow_facet setting' + } + ) + is_sortable: bool = field( + metadata={"help": "True if any of the displayed columns can be used to sort"} + ) + allow_execute_sql: bool = field( + metadata={ + "help": "True if the current actor can execute custom SQL against this database" + } + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, the one used marked with an asterisk" + } + ) + top_table: callable = field( + metadata={"help": "Async function rendering the top_table plugin slot"} + ) + count_limit: int = field( + metadata={ + "help": "The maximum number of rows Datasette will count before showing an approximation" + } + ) + + LINK_WITH_LABEL = ( '{label} {id}' ) @@ -1084,11 +1211,16 @@ async def table_view_traced(datasette, request): ) } ) + # Only keys declared on TableContext are part of the documented + # template contract - anything else in data (e.g. extras requested + # with ?_extra= on the HTML page, or extra filter context added by + # filters_from_request plugins) is dropped here + declared_fields = {f.name for f in fields(TableContext)} r = Response.html( await datasette.render_template( template, - dict( - data, + TableContext( + **{k: v for k, v in data.items() if k in declared_fields}, append_querystring=append_querystring, path_with_replaced_args=path_with_replaced_args, fix_path=datasette.urls.path, diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 3f867eb0..e9c78ecc 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -344,3 +344,28 @@ async def test_datasette_close_continues_past_db_error(): ds.close() assert good._closed assert ds._internal_database._closed + + +@pytest.mark.asyncio +async def test_datasette_render_template_dataclass_values_not_deep_copied(): + # display_rows can contain values like sqlite3.Row that cannot be + # deep-copied, so render_template must convert Context dataclasses + # shallowly - https://github.com/simonw/datasette/issues/2127 + class RefusesDeepCopy: + def __deepcopy__(self, memo): + raise RuntimeError("deepcopy not supported") + + def __str__(self): + return "shallow-copied-value" + + @dataclasses.dataclass + class ExampleContext(Context): + title: str + status: int + error: RefusesDeepCopy + + context = ExampleContext(title="Hello", status=200, error=RefusesDeepCopy()) + ds = Datasette(memory=True) + await ds.invoke_startup() + rendered = await ds.render_template("error.html", context) + assert "shallow-copied-value" in rendered diff --git a/tests/test_template_context.py b/tests/test_template_context.py index c385097a..4f08c476 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -178,6 +178,14 @@ async def test_template_context_matches_documented_contract( ) +def test_table_context_fields_match_documented_contract(): + from datasette.views.table import TableContext + + assert {f.name for f in TableContext.documented_fields()} == { + key.name for key in PAGES["table"].documented_keys() + } + + def test_base_context_keys_all_have_docs(): for key in BASE_CONTEXT_KEYS: assert key.doc, "Base context key {} is missing docs".format(key.name) From 3cc0fc07b49da3c9757ad33c73187f4fe49af557 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:01:57 -0700 Subject: [PATCH 07/20] RowContext - row page now renders a documented Context dataclass RowView declares context_class = RowContext; BaseView.render() constructs the dataclass from the assembled context, dropping any keys not declared on the class, after select_templates and alternate_url_json have been added. Extras-named fields use extra_field() so their documentation comes from the Extra classes; view-added fields carry help metadata next to the view code. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/views/base.py | 9 ++++ datasette/views/row.py | 81 +++++++++++++++++++++++++++++++++- tests/test_template_context.py | 8 ++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 2e2a5443..a3a207bd 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,5 +1,6 @@ import asyncio import csv +import dataclasses import hashlib import sys import textwrap @@ -88,6 +89,9 @@ class View: class BaseView: ds = None has_json_alternate = True + # Set to a Context subclass to render a documented template context - + # keys not declared on the class are dropped before rendering + context_class = None def __init__(self, datasette): self.ds = datasette @@ -169,6 +173,11 @@ class BaseView: ) } ) + if self.context_class is not None: + declared = {f.name for f in dataclasses.fields(self.context_class)} + template_context = self.context_class( + **{k: v for k, v in template_context.items() if k in declared} + ) return Response.html( await self.ds.render_template( template, diff --git a/datasette/views/row.py b/datasette/views/row.py index c300758b..f7475117 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -11,16 +11,95 @@ from datasette.utils import ( escape_sqlite, ) from datasette.plugins import pm +from dataclasses import dataclass, field import json import markupsafe import sqlite_utils -from datasette.extras import extra_names_from_request +from datasette.extras import extra_names_from_request, ExtraScope +from . import Context, extra_field from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry +@dataclass +class RowContext(Context): + "The page showing an individual row, e.g. /fixtures/facetable/1" + + extras_scope = ExtraScope.ROW + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + columns: list = extra_field() + database: str = extra_field() + database_color: str = extra_field() + foreign_key_tables: list = extra_field() + metadata: dict = extra_field() + primary_keys: list = extra_field() + private: bool = extra_field() + table: str = extra_field() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to value" + } + ) + primary_key_values: list = field( + metadata={"help": "Values of the primary keys for this row, from the URL"} + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + display_columns: list = field( + metadata={"help": "Column objects formatted for the HTML table display"} + ) + display_rows: list = field( + metadata={"help": "Row data formatted for the HTML table display"} + ) + custom_table_templates: list = field( + metadata={ + "help": "Custom template names that were considered for displaying this table" + } + ) + row_actions: list = field( + metadata={"help": "Row actions made available by plugin hooks"} + ) + top_row: callable = field( + metadata={"help": "Async function rendering the top_row plugin slot"} + ) + renderers: dict = field( + metadata={ + "help": "Dictionary mapping output format names (e.g. json) to their URLs for this page" + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "(name, value) pairs for hidden form fields used by the CSV export form" + } + ) + settings: dict = field( + metadata={"help": "Dictionary of Datasette's current settings"} + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, the one used marked with an asterisk" + } + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + + class RowView(DataView): name = "row" + context_class = RowContext async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4f08c476..0c694b2e 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -186,6 +186,14 @@ def test_table_context_fields_match_documented_contract(): } +def test_row_context_fields_match_documented_contract(): + from datasette.views.row import RowContext + + assert {f.name for f in RowContext.documented_fields()} == { + key.name for key in PAGES["row"].documented_keys() + } + + def test_base_context_keys_all_have_docs(): for key in BASE_CONTEXT_KEYS: assert key.doc, "Base context key {} is missing docs".format(key.name) From 8e01542fe98c237eff1390353c9c1287050f2edd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:08:21 -0700 Subject: [PATCH 08/20] One consistent pattern: every page context is a Context dataclass datasette/template_contexts.py is now a thin index with no documentation strings of its own - the docs live next to the code: - Each page's Context class (DatabaseContext, QueryContext, TableContext, RowContext) carries a docstring, its template name and help metadata on view-added fields, in the view module itself - extra_field() fields document themselves from the Extra classes - The keys render_template() adds to every page are documented in TEMPLATE_BASE_CONTEXT in app.py, next to the code that adds them, with the contract tests keeping the two in sync docs/template_context.rst is regenerated from the dataclasses, so the table and row pages now include field types like the others. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/app.py | 28 ++++ datasette/template_contexts.py | 250 ++++----------------------------- datasette/views/database.py | 8 ++ datasette/views/row.py | 3 +- datasette/views/table.py | 3 +- docs/template_context.rst | 152 ++++++++++---------- docs/template_context_doc.py | 41 +++--- tests/test_template_context.py | 78 +++------- 8 files changed, 180 insertions(+), 383 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 275baae4..b683969a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -313,6 +313,32 @@ def _to_string(value): return json.dumps(value, default=str) +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current Request object, or None", + "crumb_items": "Async function returning breadcrumb navigation items for the current page", + "urls": "Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", + "actor": "The currently authenticated actor dictionary, or None", + "menu_links": "Async function returning links for the Datasette application menu, including those added by plugins", + "display_actor": "Function returning a display string for an actor dictionary", + "show_logout": "True if the logout link should be shown in the navigation menu", + "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", + "zip": "Python's zip() builtin, made available to template logic", + "body_scripts": "List of script blocks for the page body contributed by plugins", + "format_bytes": "Function that formats a number of bytes as a human-readable size", + "show_messages": "Function returning any messages set for the current user, clearing them in the process", + "extra_css_urls": "List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", + "extra_js_urls": "List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", + "base_url": "The configured base_url setting", + "csrftoken": "Function returning the CSRF token for the current request", + "datasette_version": "The version of Datasette that is running", +} + + class Datasette: # Message constants: INFO = 1 @@ -2216,6 +2242,8 @@ class Datasette: links.extend(extra_links) return links + # Keys added here must be documented in TEMPLATE_BASE_CONTEXT - + # the contract tests fail otherwise template_context = { **context, **{ diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index e2854467..5fe6a80e 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -1,240 +1,40 @@ """ -The documented template context for Datasette's core HTML pages. +Index of the documented template contexts for Datasette's core HTML pages. -This module is the source of truth for the template context contract: -the set of variables that custom templates can rely on for each page. -It is consumed by the contract tests in tests/test_template_context.py, -which assert that the real rendered context for each page exactly -matches what is documented here, and by docs/template_context_doc.py -which generates the documentation in docs/template_context.rst. +This module deliberately contains no documentation strings of its own - +the documentation lives next to the code it describes: -Documentation for each key comes from one of three places: +- Every page renders a Context dataclass defined in its view module + (DatabaseContext, QueryContext in views/database.py, TableContext in + views/table.py, RowContext in views/row.py). Fields added by view code + carry ``help`` metadata; fields declared with extra_field() take their + documentation from the description on the matching Extra class in + views/table_extras.py. +- The keys render_template() adds to every page are documented in + TEMPLATE_BASE_CONTEXT in datasette/app.py, next to the code that adds + them. -- Pages that render a Context dataclass (database, query) use the - ``help`` metadata on each dataclass field -- Keys provided by registered extras use the ``description`` from the - Extra class -- Keys added inline by view code are documented in this module +The contract tests in tests/test_template_context.py assert that the real +rendered context for each page exactly matches what is documented, and +docs/template_context_doc.py generates docs/template_context.rst from the +same classes. """ -from dataclasses import dataclass - -from datasette.extras import ExtraScope +from datasette.app import TEMPLATE_BASE_CONTEXT from datasette.views.database import DatabaseContext, QueryContext -from datasette.views.table_extras import table_extra_registry - - -@dataclass(frozen=True) -class TemplateContextKey: - name: str - doc: str - - -def _keys(**docs): - return tuple(TemplateContextKey(name, doc) for name, doc in docs.items()) - - -# Added by Datasette.render_template() to the context for every page, -# including pages rendered by plugins -BASE_CONTEXT_KEYS = _keys( - request="The current Request object, or None", - crumb_items="Async function returning breadcrumb navigation items for the current page", - urls="Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", - actor="The currently authenticated actor dictionary, or None", - menu_links="Async function returning links for the Datasette application menu, including those added by plugins", - display_actor="Function returning a display string for an actor dictionary", - show_logout="True if the logout link should be shown in the navigation menu", - app_css_hash="Hash of Datasette's app.css contents, used for cache busting", - zip="Python's zip() builtin, made available to template logic", - body_scripts="List of script blocks for the page body contributed by plugins", - format_bytes="Function that formats a number of bytes as a human-readable size", - show_messages="Function returning any messages set for the current user, clearing them in the process", - extra_css_urls="List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", - extra_js_urls="List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", - base_url="The configured base_url setting", - csrftoken="Function returning the CSRF token for the current request", - datasette_version="The version of Datasette that is running", -) - - -@dataclass(frozen=True) -class PageContext: - # Identifier used in tests and documentation, e.g. "table" - name: str - title: str - description: str - # The default template used to render this page - template: str - # For pages rendered from a Context dataclass - context_class: type = None - # For pages whose context includes resolved extras - extras_scope: ExtraScope = None - extra_keys: tuple = () - # Keys added inline by the view code, documented here - keys: tuple = () - - def documented_keys(self): - "Every page-specific documented key, excluding BASE_CONTEXT_KEYS" - documented = [] - if self.context_class is not None: - documented.extend( - TemplateContextKey(f.name, f.help) - for f in self.context_class.documented_fields() - ) - for name in self.extra_keys: - cls = table_extra_registry.classes_by_name[name] - documented.append(TemplateContextKey(name, cls.description or "")) - documented.extend(self.keys) - return sorted(documented, key=lambda key: key.name) - - -_SHARED_DOCS = dict( - ok="True if the data for this page was retrieved without errors", - rows="The rows for this page, as a list of dictionaries mapping column name to value", - query_ms="Time taken by the SQL queries for this page, in milliseconds", - select_templates="List of template names that were considered for this page, the one used marked with an asterisk", - settings="Dictionary of Datasette's current settings", - alternate_url_json="URL for the JSON version of this page", - url_csv="URL for the CSV export of this page", - url_csv_path="Path portion of the CSV export URL", - url_csv_hidden_args="(name, value) pairs for hidden form fields used by the CSV export form", - renderers="Dictionary mapping output format names (e.g. json) to their URLs for this page", - display_columns="Column objects formatted for the HTML table display", - display_rows="Row data formatted for the HTML table display", - custom_table_templates="Custom template names that were considered for displaying this table", -) - - -def _shared(*names): - return tuple(TemplateContextKey(name, _SHARED_DOCS[name]) for name in names) - +from datasette.views.row import RowContext +from datasette.views.table import TableContext PAGES = { - page.name: page - for page in ( - PageContext( - name="database", - title="Database", - description="The page listing the tables, views and queries in a database, e.g. /fixtures.", - template="database.html", - context_class=DatabaseContext, - ), - PageContext( - name="query", - title="Query", - description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name).", - template="query.html", - context_class=QueryContext, - ), - PageContext( - name="table", - title="Table", - description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable.", - template="table.html", - extras_scope=ExtraScope.TABLE, - extra_keys=( - "actions", - "all_columns", - "columns", - "count", - "count_sql", - "custom_table_templates", - "database", - "database_color", - "display_columns", - "display_rows", - "expandable_columns", - "facet_results", - "facets_timed_out", - "filters", - "form_hidden_args", - "human_description_en", - "is_view", - "metadata", - "next_url", - "primary_keys", - "private", - "query", - "renderers", - "set_column_type_ui", - "sorted_facet_results", - "suggested_facets", - "table", - "table_definition", - "view_definition", - ), - keys=_shared( - "ok", - "rows", - "query_ms", - "select_templates", - "settings", - "alternate_url_json", - "url_csv", - "url_csv_path", - "url_csv_hidden_args", - ) - + _keys( - allow_execute_sql="True if the current actor can execute custom SQL against this database", - append_querystring="Function that appends additional querystring arguments to a URL", - count_limit="The maximum number of rows Datasette will count before showing an approximation", - datasette_allow_facet='The string "true" or "false" reflecting the allow_facet setting', - extra_wheres_for_ui="Extra where clauses from ?_where=, with links to remove them", - filter_columns="List of columns offered by the filter interface", - fix_path="Function that applies the base_url prefix to a path", - is_sortable="True if any of the displayed columns can be used to sort", - next="Pagination token for the next page, or None", - path_with_replaced_args="Function for building the current path with modified querystring arguments", - sort="Column the page is sorted by, or None", - sort_desc="Column the page is sorted by in descending order, or None", - supports_search="True if this table has full-text search configured", - top_table="Async function rendering the top_table plugin slot", - ), - ), - PageContext( - name="row", - title="Row", - description="The page showing an individual row, e.g. /fixtures/facetable/1.", - template="row.html", - extras_scope=ExtraScope.ROW, - extra_keys=( - "columns", - "database", - "database_color", - "foreign_key_tables", - "metadata", - "primary_keys", - "private", - "table", - ), - keys=_shared( - "ok", - "rows", - "query_ms", - "select_templates", - "settings", - "alternate_url_json", - "url_csv", - "url_csv_path", - "url_csv_hidden_args", - "renderers", - "display_columns", - "display_rows", - "custom_table_templates", - ) - + _keys( - primary_key_values="Values of the primary keys for this row, from the URL", - row_actions="Row actions made available by plugin hooks", - top_row="Async function rendering the top_row plugin slot", - ), - ), - ) + "database": DatabaseContext, + "query": QueryContext, + "table": TableContext, + "row": RowContext, } def documented_context_keys(page_name): "Set of every documented key for the named page, including base context keys" - page = PAGES[page_name] - return {key.name for key in BASE_CONTEXT_KEYS} | { - key.name for key in page.documented_keys() + return set(TEMPLATE_BASE_CONTEXT) | { + f.name for f in PAGES[page_name].documented_fields() } diff --git a/datasette/views/database.py b/datasette/views/database.py index f1756863..4f05c804 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -228,6 +228,10 @@ class DatabaseView(View): @dataclass class DatabaseContext(Context): + "The page listing the tables, views and queries in a database, e.g. /fixtures." + + template = "database.html" + database: str = field(metadata={"help": "The name of the database"}) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -281,6 +285,10 @@ class DatabaseContext(Context): @dataclass class QueryContext(Context): + "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." + + template = "query.html" + database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( diff --git a/datasette/views/row.py b/datasette/views/row.py index f7475117..3ce02e21 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -23,8 +23,9 @@ from .table_extras import RowExtraContext, resolve_row_extras, table_extra_regis @dataclass class RowContext(Context): - "The page showing an individual row, e.g. /fixtures/facetable/1" + "The page showing an individual row, e.g. /fixtures/facetable/1." + template = "row.html" extras_scope = ExtraScope.ROW # Fields resolved by registered extras - their documentation comes diff --git a/datasette/views/table.py b/datasette/views/table.py index 356247ff..8ad7fa8c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -59,8 +59,9 @@ from .table_extras import ( @dataclass class TableContext(Context): - "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable" + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." + template = "table.html" extras_scope = ExtraScope.TABLE # Fields resolved by registered extras - their documentation comes diff --git a/docs/template_context.rst b/docs/template_context.rst index a487a6e7..e0a34921 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -250,160 +250,160 @@ The page showing the rows in a table or SQL view, e.g. /fixtures/facetable. Rend Many of these keys are shared with the :ref:`JSON API ` for this page. -``actions`` +``actions`` - ``callable`` Table or view actions made available by plugin hooks -``all_columns`` +``all_columns`` - ``list`` All columns in the table, regardless of _col/_nocol filtering -``allow_execute_sql`` +``allow_execute_sql`` - ``bool`` True if the current actor can execute custom SQL against this database -``alternate_url_json`` +``alternate_url_json`` - ``str`` URL for the JSON version of this page -``append_querystring`` +``append_querystring`` - ``callable`` Function that appends additional querystring arguments to a URL -``columns`` +``columns`` - ``list`` Column names returned by this query -``count`` +``count`` - ``int`` Total count of rows matching these filters -``count_limit`` +``count_limit`` - ``int`` The maximum number of rows Datasette will count before showing an approximation -``count_sql`` +``count_sql`` - ``str`` SQL query used to calculate the total count -``custom_table_templates`` +``custom_table_templates`` - ``list`` Custom template names considered for this table -``database`` +``database`` - ``str`` Database name -``database_color`` +``database_color`` - ``str`` Color assigned to the database -``datasette_allow_facet`` +``datasette_allow_facet`` - ``str`` The string "true" or "false" reflecting the allow_facet setting -``display_columns`` +``display_columns`` - ``list`` Column metadata used by the HTML table display -``display_rows`` +``display_rows`` - ``list`` Row data formatted for the HTML table display -``expandable_columns`` +``expandable_columns`` - ``list`` Foreign key columns that can be expanded with labels -``extra_wheres_for_ui`` +``extra_wheres_for_ui`` - ``list`` Extra where clauses from ?_where=, with links to remove them -``facet_results`` +``facet_results`` - ``dict`` Results of facets calculated against this data -``facets_timed_out`` +``facets_timed_out`` - ``list`` Facet calculations that timed out -``filter_columns`` +``filter_columns`` - ``list`` List of columns offered by the filter interface -``filters`` +``filters`` - ``Filters`` Filters object used by the HTML table interface -``fix_path`` +``fix_path`` - ``callable`` Function that applies the base_url prefix to a path -``form_hidden_args`` +``form_hidden_args`` - ``list`` Hidden form arguments used by the HTML table interface -``human_description_en`` +``human_description_en`` - ``str`` Human-readable description of the filters -``is_sortable`` +``is_sortable`` - ``bool`` True if any of the displayed columns can be used to sort -``is_view`` +``is_view`` - ``bool`` Whether this resource is a view instead of a table -``metadata`` +``metadata`` - ``dict`` Metadata about the table, database or stored query -``next`` +``next`` - ``str`` Pagination token for the next page, or None -``next_url`` +``next_url`` - ``str`` Full URL for the next page of results -``ok`` +``ok`` - ``bool`` True if the data for this page was retrieved without errors -``path_with_replaced_args`` +``path_with_replaced_args`` - ``callable`` Function for building the current path with modified querystring arguments -``primary_keys`` +``primary_keys`` - ``list`` Primary keys for this table -``private`` +``private`` - ``bool`` Whether this resource is private to the current actor -``query`` +``query`` - ``dict`` Details of the underlying SQL query -``query_ms`` +``query_ms`` - ``float`` Time taken by the SQL queries for this page, in milliseconds -``renderers`` +``renderers`` - ``dict`` Alternative output renderers available for this table -``rows`` +``rows`` - ``list`` The rows for this page, as a list of dictionaries mapping column name to value -``select_templates`` +``select_templates`` - ``list`` List of template names that were considered for this page, the one used marked with an asterisk -``set_column_type_ui`` +``set_column_type_ui`` - ``dict`` Column type UI metadata for this table -``settings`` +``settings`` - ``dict`` Dictionary of Datasette's current settings -``sort`` +``sort`` - ``str`` Column the page is sorted by, or None -``sort_desc`` +``sort_desc`` - ``str`` Column the page is sorted by in descending order, or None -``sorted_facet_results`` +``sorted_facet_results`` - ``list`` Facet results sorted for display -``suggested_facets`` +``suggested_facets`` - ``list`` Suggestions for facets that might return interesting results -``supports_search`` +``supports_search`` - ``bool`` True if this table has full-text search configured -``table`` +``table`` - ``str`` Table name -``table_definition`` +``table_definition`` - ``str`` SQL definition for this table -``top_table`` +``top_table`` - ``callable`` Async function rendering the top_table plugin slot -``url_csv`` +``url_csv`` - ``str`` URL for the CSV export of this page -``url_csv_hidden_args`` +``url_csv_hidden_args`` - ``list`` (name, value) pairs for hidden form fields used by the CSV export form -``url_csv_path`` +``url_csv_path`` - ``str`` Path portion of the CSV export URL -``view_definition`` +``view_definition`` - ``str`` SQL definition for this view Row page @@ -413,76 +413,76 @@ The page showing an individual row, e.g. /fixtures/facetable/1. Rendered using t Many of these keys are shared with the :ref:`JSON API ` for this page. -``alternate_url_json`` +``alternate_url_json`` - ``str`` URL for the JSON version of this page -``columns`` +``columns`` - ``list`` Column names returned by this query -``custom_table_templates`` +``custom_table_templates`` - ``list`` Custom template names that were considered for displaying this table -``database`` +``database`` - ``str`` Database name -``database_color`` +``database_color`` - ``str`` Color assigned to the database -``display_columns`` +``display_columns`` - ``list`` Column objects formatted for the HTML table display -``display_rows`` +``display_rows`` - ``list`` Row data formatted for the HTML table display -``foreign_key_tables`` +``foreign_key_tables`` - ``list`` Tables that link to this row using foreign keys -``metadata`` +``metadata`` - ``dict`` Metadata about the table, database or stored query -``ok`` +``ok`` - ``bool`` True if the data for this page was retrieved without errors -``primary_key_values`` +``primary_key_values`` - ``list`` Values of the primary keys for this row, from the URL -``primary_keys`` +``primary_keys`` - ``list`` Primary keys for this table -``private`` +``private`` - ``bool`` Whether this resource is private to the current actor -``query_ms`` +``query_ms`` - ``float`` Time taken by the SQL queries for this page, in milliseconds -``renderers`` +``renderers`` - ``dict`` Dictionary mapping output format names (e.g. json) to their URLs for this page -``row_actions`` +``row_actions`` - ``list`` Row actions made available by plugin hooks -``rows`` +``rows`` - ``list`` The rows for this page, as a list of dictionaries mapping column name to value -``select_templates`` +``select_templates`` - ``list`` List of template names that were considered for this page, the one used marked with an asterisk -``settings`` +``settings`` - ``dict`` Dictionary of Datasette's current settings -``table`` +``table`` - ``str`` Table name -``top_row`` +``top_row`` - ``callable`` Async function rendering the top_row plugin slot -``url_csv`` +``url_csv`` - ``str`` URL for the CSV export of this page -``url_csv_hidden_args`` +``url_csv_hidden_args`` - ``list`` (name, value) pairs for hidden form fields used by the CSV export form -``url_csv_path`` +``url_csv_path`` - ``str`` Path portion of the CSV export URL .. [[[end]]] diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index 1539dc5e..c3ec77e2 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -1,11 +1,12 @@ """ -Cog helpers for generating docs/template_context.rst from the manifest -in datasette/template_contexts.py - same pattern as json_api_doc.py. +Cog helpers for generating docs/template_context.rst from the Context +dataclasses and TEMPLATE_BASE_CONTEXT - same pattern as json_api_doc.py. """ def template_context(cog): - from datasette.template_contexts import BASE_CONTEXT_KEYS, PAGES + from datasette.app import TEMPLATE_BASE_CONTEXT + from datasette.template_contexts import PAGES cog.out("\n") _section( @@ -19,36 +20,26 @@ def template_context(cog): ":ref:`plugin_hook_extra_template_vars` hook." ), ) - _untyped_keys(cog, BASE_CONTEXT_KEYS) + for name, doc in TEMPLATE_BASE_CONTEXT.items(): + cog.out("``{}``\n".format(name)) + cog.out(" {}\n\n".format(doc)) - for page in PAGES.values(): - _section( - cog, - "{} page".format(page.title), - "{} Rendered using the ``{}`` template.".format( - page.description, page.template - ), + for klass in PAGES.values(): + title = "{} page".format(klass.__name__.removesuffix("Context")) + intro = "{} Rendered using the ``{}`` template.".format( + klass.__doc__, klass.template ) - if page.context_class is not None: - for f in sorted( - page.context_class.documented_fields(), key=lambda f: f.name - ): - cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) - cog.out(" {}\n\n".format(f.help)) - else: + _section(cog, title, intro) + if klass.extras_scope is not None: cog.out( "Many of these keys are shared with the :ref:`JSON API " "` for this page.\n\n" ) - _untyped_keys(cog, page.documented_keys()) + for f in sorted(klass.documented_fields(), key=lambda f: f.name): + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) def _section(cog, title, intro): cog.out("{}\n{}\n\n".format(title, "-" * len(title))) cog.out("{}\n\n".format(intro)) - - -def _untyped_keys(cog, keys): - for key in keys: - cog.out("``{}``\n".format(key.name)) - cog.out(" {}\n\n".format(key.doc)) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 0c694b2e..b970cb8a 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,17 +10,11 @@ from dataclasses import dataclass, field import pytest -from datasette.app import Datasette +from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database -from datasette.template_contexts import ( - BASE_CONTEXT_KEYS, - PAGES, - documented_context_keys, -) +from datasette.template_contexts import PAGES, documented_context_keys from datasette.views import Context -from datasette.views.database import DatabaseContext, QueryContext -from datasette.views.table_extras import table_extra_registry def test_documented_fields(): @@ -36,14 +30,20 @@ def test_documented_fields(): ] -@pytest.mark.parametrize("klass", (DatabaseContext, QueryContext)) -def test_context_dataclass_fields_all_have_help(klass): +@pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) +def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert context_field.help, "{}.{} is missing help metadata".format( + assert context_field.help, "{}.{} is missing documentation".format( klass.__name__, context_field.name ) +@pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) +def test_context_class_has_docstring_and_template(klass): + assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) + assert klass.template, "{} is missing a template".format(klass.__name__) + + def test_extra_field_documentation_comes_from_the_extra_class(): from datasette.views import extra_field from datasette.views.table_extras import CountExtra @@ -169,8 +169,8 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - "Undocumented keys in {} template context: {} - document them in " - "datasette/template_contexts.py".format(page_name, sorted(undocumented)) + "Undocumented keys in {} template context: {} - add them to the " + "page's Context class".format(page_name, sorted(undocumented)) ) assert not no_longer_present, ( "Documented keys missing from {} template context: {} - this would " @@ -178,53 +178,21 @@ async def test_template_context_matches_documented_contract( ) -def test_table_context_fields_match_documented_contract(): - from datasette.views.table import TableContext - - assert {f.name for f in TableContext.documented_fields()} == { - key.name for key in PAGES["table"].documented_keys() - } - - -def test_row_context_fields_match_documented_contract(): - from datasette.views.row import RowContext - - assert {f.name for f in RowContext.documented_fields()} == { - key.name for key in PAGES["row"].documented_keys() - } - - def test_base_context_keys_all_have_docs(): - for key in BASE_CONTEXT_KEYS: - assert key.doc, "Base context key {} is missing docs".format(key.name) - - -@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) -def test_page_documented_keys_all_have_docs(page): - for key in page.documented_keys(): - assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) + for name, doc in TEMPLATE_BASE_CONTEXT.items(): + assert doc, "Base context key {} is missing docs".format(name) def test_template_context_docs_cover_every_documented_key(): docs_path = pathlib.Path(__file__).parent.parent / "docs" / "template_context.rst" assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() - for key in BASE_CONTEXT_KEYS: - assert "``{}``".format(key.name) in docs, key.name - for page in PAGES.values(): - assert page.title in docs, page.title - for key in page.documented_keys(): - assert "``{}``".format(key.name) in docs, "{} ({} page)".format( - key.name, page.name + for name in TEMPLATE_BASE_CONTEXT: + assert "``{}``".format(name) in docs, name + for page_name, klass in PAGES.items(): + title = "{} page".format(klass.__name__.removesuffix("Context")) + assert title in docs, title + for context_field in klass.documented_fields(): + assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( + context_field.name, page_name ) - - -@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) -def test_page_extra_keys_are_registered_extras(page): - for name in page.extra_keys: - cls = table_extra_registry.classes_by_name.get(name) - assert cls is not None, "{} is not a registered extra".format(name) - assert page.extras_scope is not None - assert cls.available_for( - page.extras_scope - ), "{} extra is not available for scope {}".format(name, page.extras_scope) From 4d031c85621ab7e63b1952d25e3e938fa72323b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 19:47:21 -0700 Subject: [PATCH 09/20] Add count_truncated template context --- datasette/templates/database.html | 2 +- datasette/templates/table.html | 4 ++-- datasette/views/__init__.py | 2 ++ datasette/views/database.py | 29 ++++++++++++++++++++++---- datasette/views/table.py | 33 +++++++++++++++++++++++------- datasette/views/table_extras.py | 4 ++-- docs/template_context.rst | 13 +++++------- tests/test_template_context.py | 34 ++++++++++++++++++++++++++++++- 8 files changed, 96 insertions(+), 25 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 23eeb571..c09582f7 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -76,7 +76,7 @@

{{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

-

{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

+

{% if table.count is none %}Many rows{% elif table.count_truncated %}>{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

{% endif %} {% endfor %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index e06ef94e..b7b776ab 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -48,7 +48,7 @@ {% if count or human_description_en %}

- {% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows + {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %} diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 238b9bb3..e503ed35 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -32,6 +32,8 @@ class Context: "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + if f.name.startswith("_"): + continue from_extra = bool(f.metadata.get("from_extra")) if from_extra: help_text = cls._extra_description(f.name) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0d35b87f..46a90c4c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -230,7 +230,6 @@ class DatabaseView(View): database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, - count_limit=db.count_limit, allow_download=datasette.setting("allow_download") and not db.is_mutable and not db.is_memory, @@ -267,7 +266,11 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of stored query objects"}) @@ -295,7 +298,6 @@ class DatabaseContext(Context): editable: bool = field( metadata={"help": "Boolean indicating if the database is editable"} ) - count_limit: int = field(metadata={"help": "The maximum number of rows to count"}) allow_download: bool = field( metadata={"help": "Boolean indicating if database download is allowed"} ) @@ -365,7 +367,11 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) named_parameter_values: dict = field( metadata={"help": "Dictionary of parameter names/values"} ) @@ -430,6 +436,9 @@ async def get_tables(datasette, request, db, allowed_dict): "columns": table_columns, "primary_keys": await db.primary_keys(table), "count": table_counts[table], + "count_truncated": _table_count_truncated( + datasette, db, table, table_counts[table] + ), "hidden": table in hidden_table_names, "fts_table": await db.fts_table(table), "foreign_keys": all_foreign_keys[table], @@ -440,6 +449,18 @@ async def get_tables(datasette, request, db, allowed_dict): return tables +def _table_count_truncated(datasette, db, table, count): + if count != db.count_limit + 1: + return False + if not db.is_mutable and datasette.inspect_data: + try: + datasette.inspect_data[db.name]["tables"][table]["count"] + return False + except KeyError: + pass + return True + + async def database_download(request, datasette): from datasette.resources import DatabaseResource diff --git a/datasette/views/table.py b/datasette/views/table.py index a6769242..38a69f5f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -113,6 +113,11 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + count_truncated: bool = field( + metadata={ + "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." + } + ) rows: list = field( metadata={ "help": "The rows for this page, as a list of dictionaries mapping column name to value" @@ -185,11 +190,6 @@ class TableContext(Context): top_table: callable = field( metadata={"help": "Async function rendering the top_table plugin slot"} ) - count_limit: int = field( - metadata={ - "help": "The maximum number of rows Datasette will count before showing an approximation" - } - ) table_page_data: dict = field( metadata={"help": "JSON data used by JavaScript on the table page"} ) @@ -1391,7 +1391,6 @@ class TableFragmentView(BaseView): path_with_replaced_args=path_with_replaced_args, fix_path=self.ds.urls.path, settings=self.ds.settings_dict(), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1843,7 +1842,6 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -2276,6 +2274,9 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: + data["count_truncated"] = _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, data.get("count") + ) data.update(extra_context_from_filters) # filter_columns combine the columns we know are available # in the table with any additional columns (such as rowid) @@ -2341,6 +2342,24 @@ async def table_view_data( return data, rows[:page_size], columns, expanded_columns, sql, next_url +def _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, count +): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 7cb4d8f0..f688433d 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -127,8 +127,8 @@ class CountExtra(Extra): pass if context.count_sql and count is None and not context.nocount: - count_sql_limited = ( - f"select count(*) from (select * {context.from_sql} limit 10001)" + count_sql_limited = "select count(*) from (select * {} limit {})".format( + context.from_sql, context.db.count_limit + 1 ) try: count_rows = list( diff --git a/docs/template_context.rst b/docs/template_context.rst index f94ecd48..b27f2f79 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -100,9 +100,6 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``attached_databases`` - ``list`` List of names of attached databases -``count_limit`` - ``int`` - The maximum number of rows to count - ``database`` - ``str`` The name of the database @@ -152,7 +149,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re Dictionary mapping table names to their column lists ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Callable to render the top_database slot @@ -241,7 +238,7 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie Dictionary of table name to list of column names ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Callable to render the top_query slot @@ -280,12 +277,12 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. ``count`` - ``int`` Total count of rows matching these filters -``count_limit`` - ``int`` - The maximum number of rows Datasette will count before showing an approximation - ``count_sql`` - ``str`` SQL query used to calculate the total count +``count_truncated`` - ``bool`` + True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit. + ``custom_table_templates`` - ``list`` Custom template names considered for this table diff --git a/tests/test_template_context.py b/tests/test_template_context.py index b970cb8a..8d78bf77 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -21,6 +21,7 @@ def test_documented_fields(): @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) + _internal: str = field() count: int = field(metadata={"help": "How many there are"}) fields = DemoContext.documented_fields() @@ -165,7 +166,11 @@ async def test_template_context_matches_documented_contract( # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context documented = documented_context_keys(page_name) - actual = set(await get_template_context(context_ds, path)) + actual = { + key + for key in await get_template_context(context_ds, path) + if not key.startswith("_") + } undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( @@ -178,6 +183,33 @@ async def test_template_context_matches_documented_contract( ) +@pytest.mark.asyncio +async def test_count_truncated_replaces_count_limit_context_key(context_ds): + db = context_ds.databases["fixtures"] + previous_count_limit = db.count_limit + previous_cached_table_counts = db._cached_table_counts + db.count_limit = 10 + db._cached_table_counts = None + try: + table_context = await get_template_context(context_ds, "/fixtures/facetable") + assert table_context["count"] == 11 + assert table_context["count_truncated"] is True + assert "count_limit" not in table_context + + database_context = await get_template_context(context_ds, "/fixtures") + facetable = next( + table + for table in database_context["tables"] + if table["name"] == "facetable" + ) + assert facetable["count"] == 11 + assert facetable["count_truncated"] is True + assert "count_limit" not in database_context + finally: + db.count_limit = previous_count_limit + db._cached_table_counts = previous_cached_table_counts + + def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name) From 29971d97293a78ca44f66a1129d8c8b922b09c38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 07:29:14 -0700 Subject: [PATCH 10/20] Clarify base template context docs --- datasette/app.py | 42 +++++++++++++++++++++------------------ docs/template_context.rst | 29 ++++++++++++--------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 9f0c8397..90de60a9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -35,6 +35,7 @@ from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, + pass_context, PrefixLoader, ) from jinja2.environment import Template @@ -330,30 +331,37 @@ def _to_string(value): return json.dumps(value, default=str) +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + # Documentation for the variables Datasette.render_template() adds to the # context for every page. This is part of the documented template contract: # keys added in render_template() must be documented here - the contract # tests in tests/test_template_context.py enforce this, and the docs in # docs/template_context.rst are generated from it. TEMPLATE_BASE_CONTEXT = { - "request": "The current Request object, or None", - "crumb_items": "Async function returning breadcrumb navigation items for the current page", - "urls": "Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", - "actor": "The currently authenticated actor dictionary, or None", - "menu_links": "Async function returning links for the Datasette application menu, including those added by plugins", - "display_actor": "Function returning a display string for an actor dictionary", + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", "show_logout": "True if the logout link should be shown in the navigation menu", "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", "edit_tools_js_hash": "Hash of Datasette's edit-tools.js contents, used for cache busting", "table_js_hash": "Hash of Datasette's table.js contents, used for cache busting", - "zip": "Python's zip() builtin, made available to template logic", - "body_scripts": "List of script blocks for the page body contributed by plugins", - "format_bytes": "Function that formats a number of bytes as a human-readable size", - "show_messages": "Function returning any messages set for the current user, clearing them in the process", - "extra_css_urls": "List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", - "extra_js_urls": "List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", - "base_url": "The configured base_url setting", - "csrftoken": "Function returning the CSRF token for the current request", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - """.format(escape(ex.sql))).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except (sqlite3.OperationalError, InvalidSql) as e: - raise DatasetteError(str(e), title="Invalid SQL", status=400) - - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) - - except DatasetteError: - raise - - end = time.perf_counter() - data["query_ms"] = (end - start) * 1000 - - # Special case for .jsono extension - redirect to _shape=objects - if _format == "jsono": - return self.redirect( - request, - path_with_added_args( - request, - {"_shape": "objects"}, - path=request.path.rsplit(".jsono", 1)[0] + ".json", - ), - forward_querystring=False, - ) - - if _format in self.ds.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - self.ds.renderers[_format][0], - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - truncated=False, # TODO: support this - error=data.get("error"), - # These will be deprecated in Datasette 1.0: - args=request.args, - data=data, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code", status_code or 200), - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), - ) - elif isinstance(result, Response): - r = result - if status_code is not None: - # Over-ride the status code - r.status = status_code - else: - assert False, f"{result} should be dict or Response" - else: - extras = {} - if callable(extra_template_data): - extras = extra_template_data() - if asyncio.iscoroutine(extras): - extras = await extras - else: - extras = extra_template_data - url_labels_extra = {} - if data.get("expandable_columns"): - url_labels_extra = {"_labels": "on"} - - renderers = {} - for key, (_, can_render) in self.ds.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - - url_csv_args = {"_size": "max", **url_labels_extra} - url_csv = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format="csv", - extra_qs=url_csv_args, - ) - ) - url_csv_path = url_csv.split("?")[0] - context = { - **data, - **extras, - **{ - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - }, - } - if "metadata" not in context: - context["metadata"] = await self.ds.get_instance_metadata() - r = await self.render(templates, request=request, context=context) - if status_code is not None: - r.status = status_code - - ttl = request.args.get("_ttl", None) - if ttl is None or not ttl.isdigit(): - ttl = self.ds.setting("default_cache_ttl") - - return self.set_response_headers(r, ttl) - - def set_response_headers(self, response, ttl): - # Set far-future cache expiry - if self.ds.cache_headers and response.status == 200: - ttl = int(ttl) - if ttl == 0: - ttl_header = "no-cache" - else: - ttl_header = f"max-age={ttl}" - response.headers["Cache-Control"] = ttl_header - response.headers["Referrer-Policy"] = "no-referrer" - if self.ds.cors: - add_cors_headers(response.headers) - return response - - def _error(messages, status=400): return Response.json({"ok": False, "errors": messages}, status=status) diff --git a/datasette/views/row.py b/datasette/views/row.py index 7802f45e..3e3e52a9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,21 +1,34 @@ +import asyncio +import json +import textwrap +import time +import urllib.parse +from dataclasses import dataclass, field, fields + +import markupsafe +import sqlite_utils + from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource -from .base import DataView, BaseView, _error +from .base import BaseView, DatasetteError, _error, stream_csv from datasette.utils import ( + add_cors_headers, await_me_maybe, + call_with_supported_arguments, CustomRow, + InvalidSql, make_slot_function, path_from_row_pks, + path_with_added_args, + path_with_format, + path_with_removed_args, to_css_class, escape_sqlite, + sqlite3, ) from datasette.plugins import pm -from dataclasses import dataclass, field -import json -import markupsafe -import sqlite_utils from datasette.extras import extra_names_from_request, ExtraScope from . import Context, extra_field from .table import ( @@ -121,9 +134,259 @@ class RowContext(Context): ) -class RowView(DataView): +class RowView(BaseView): name = "row" - context_class = RowContext + + def redirect(self, request, path, forward_querystring=True, remove_args=None): + if request.query_string and "?" not in path and forward_querystring: + path = f"{path}?{request.query_string}" + if remove_args: + path = path_with_removed_args(request, remove_args, path=path) + response = Response.redirect(path) + response.headers["Link"] = f"<{path}>; rel=preload" + if self.ds.cors: + add_cors_headers(response.headers) + return response + + async def as_csv(self, request, database): + return await stream_csv(self.ds, self.data, request, database) + + async def get(self, request): + db = await self.ds.resolve_database(request) + database = db.name + database_route = db.route + format_ = request.url_vars.get("format") or "html" + data_kwargs = {} + + if format_ == "csv": + return await self.as_csv(request, database_route) + + if format_ == "html": + # HTML views default to expanding all foreign key labels + data_kwargs["default_labels"] = True + + extra_template_data = {} + start = time.perf_counter() + status_code = None + templates = () + try: + response_or_template_contexts = await self.data(request, **data_kwargs) + if isinstance(response_or_template_contexts, Response): + return response_or_template_contexts + # If it has four items, it includes an HTTP status code + if len(response_or_template_contexts) == 4: + ( + data, + extra_template_data, + templates, + status_code, + ) = response_or_template_contexts + else: + data, extra_template_data, templates = response_or_template_contexts + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent(""" +

SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

+ + + """.format(markupsafe.escape(ex.sql))).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except (sqlite3.OperationalError, InvalidSql) as e: + raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: + raise + + end = time.perf_counter() + data["query_ms"] = (end - start) * 1000 + + # Special case for .jsono extension - redirect to _shape=objects + if format_ == "jsono": + return self.redirect( + request, + path_with_added_args( + request, + {"_shape": "objects"}, + path=request.path.rsplit(".jsono", 1)[0] + ".json", + ), + forward_querystring=False, + ) + + if format_ in self.ds.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + self.ds.renderers[format_][0], + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + truncated=False, # TODO: support this + error=data.get("error"), + # These will be deprecated in Datasette 1.0: + args=request.args, + data=data, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + response = Response( + body=result.get("body"), + status=result.get("status_code", status_code or 200), + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + response = result + if status_code is not None: + # Over-ride the status code + response.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + response = await self.html(request, data, extra_template_data, templates) + if status_code is not None: + response.status = status_code + else: + raise NotFound("Invalid format: {}".format(format_)) + + ttl = request.args.get("_ttl", None) + if ttl is None or not ttl.isdigit(): + ttl = self.ds.setting("default_cache_ttl") + + return self.set_response_headers(response, ttl) + + async def html(self, request, data, extra_template_data, templates): + extras = {} + if callable(extra_template_data): + extras = extra_template_data() + if asyncio.iscoroutine(extras): + extras = await extras + else: + extras = extra_template_data + + url_labels_extra = {} + if data.get("expandable_columns"): + url_labels_extra = {"_labels": "on"} + + renderers = {} + for key, (_, can_render) in self.ds.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=data.get("database"), + table=data.get("table"), + request=request, + view_name=self.name, + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + + url_csv_args = {"_size": "max", **url_labels_extra} + url_csv = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="csv", + extra_qs=url_csv_args, + ) + ) + url_csv_path = url_csv.split("?")[0] + context = {**data, **extras} + if "metadata" not in context: + context["metadata"] = await self.ds.get_instance_metadata() + + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="json", + ) + ), + ) + explicit_context = { + "renderers": renderers, + "url_csv": url_csv, + "url_csv_path": url_csv_path, + "url_csv_hidden_args": [ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + "settings": self.ds.settings_dict(), + "alternate_url_json": alternate_url_json, + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + } + declared_fields = {f.name for f in fields(RowContext)} + context = { + key: value + for key, value in context.items() + if key in declared_fields and key not in explicit_context + } + + return Response.html( + await self.ds.render_template( + template, + RowContext(**context, **explicit_context), + request=request, + view_name=self.name, + ), + headers={ + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, + ) + + def set_response_headers(self, response, ttl): + # Set far-future cache expiry + if self.ds.cache_headers and response.status == 200: + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" + else: + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header + response.headers["Referrer-Policy"] = "no-referrer" + if self.ds.cors: + add_cors_headers(response.headers) + return response async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) From a43e76c31a3c4b96ab7289b5f3202c7c3537bf6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 09:08:40 -0700 Subject: [PATCH 15/20] Construct row template context explicitly --- datasette/views/row.py | 63 +++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3e3e52a9..ef39dce6 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -3,7 +3,7 @@ import json import textwrap import time import urllib.parse -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field import markupsafe import sqlite_utils @@ -336,34 +336,45 @@ class RowView(BaseView): ) ), ) - explicit_context = { - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - "alternate_url_json": alternate_url_json, - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - } - declared_fields = {f.name for f in fields(RowContext)} - context = { - key: value - for key, value in context.items() - if key in declared_fields and key not in explicit_context - } - return Response.html( await self.ds.render_template( template, - RowContext(**context, **explicit_context), + RowContext( + columns=context["columns"], + database=context["database"], + database_color=context["database_color"], + foreign_key_tables=context["foreign_key_tables"], + metadata=context["metadata"], + primary_keys=context["primary_keys"], + private=context["private"], + table=context["table"], + ok=context["ok"], + rows=context["rows"], + primary_key_values=context["primary_key_values"], + query_ms=context["query_ms"], + display_columns=context["display_columns"], + display_rows=context["display_rows"], + custom_table_templates=context["custom_table_templates"], + row_actions=context["row_actions"], + row_mutation_ui=context["row_mutation_ui"], + table_page_data=context["table_page_data"], + top_row=context["top_row"], + renderers=renderers, + url_csv=url_csv, + url_csv_path=url_csv_path, + url_csv_hidden_args=[ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + settings=self.ds.settings_dict(), + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + alternate_url_json=alternate_url_json, + ), request=request, view_name=self.name, ), From 59ab0c0ca0b2d86913102dace6813f42f947d772 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 11:30:30 -0700 Subject: [PATCH 16/20] Clarify template context metadata names --- datasette/template_contexts.py | 2 +- datasette/views/__init__.py | 14 ++++---- datasette/views/database.py | 4 +-- datasette/views/row.py | 20 +++++------ datasette/views/table.py | 62 +++++++++++++++++----------------- docs/template_context_doc.py | 2 +- tests/test_template_context.py | 24 +++++++------ 7 files changed, 65 insertions(+), 63 deletions(-) diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index 5fe6a80e..232eb051 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -7,7 +7,7 @@ the documentation lives next to the code it describes: - Every page renders a Context dataclass defined in its view module (DatabaseContext, QueryContext in views/database.py, TableContext in views/table.py, RowContext in views/row.py). Fields added by view code - carry ``help`` metadata; fields declared with extra_field() take their + carry ``help`` metadata; fields declared with from_extra() take their documentation from the description on the matching Extra class in views/table_extras.py. - The keys render_template() adds to every page are documented in diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index e503ed35..6fba1c90 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -10,7 +10,7 @@ class ContextField: from_extra: bool = False -def extra_field(): +def from_extra(): """ Declare a Context dataclass field whose value comes from a registered Extra of the same name - its documentation is the Extra description, @@ -23,7 +23,7 @@ def extra_field(): class Context: "Base class for all documented contexts" - # Set on subclasses whose extra_field() fields should be resolved + # Set on subclasses whose from_extra() fields should be resolved # against the extras registry for this scope extras_scope = None @@ -34,8 +34,8 @@ class Context: for f in dataclasses.fields(cls): if f.name.startswith("_"): continue - from_extra = bool(f.metadata.get("from_extra")) - if from_extra: + is_from_extra = bool(f.metadata.get("from_extra")) + if is_from_extra: help_text = cls._extra_description(f.name) else: help_text = f.metadata.get("help", "") @@ -44,7 +44,7 @@ class Context: name=f.name, type_name=getattr(f.type, "__name__", str(f.type)), help=help_text, - from_extra=from_extra, + from_extra=is_from_extra, ) ) return documented @@ -59,14 +59,14 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - "{}.{} is declared with extra_field() but there is no " + "{}.{} is declared with from_extra() but there is no " "registered extra of that name".format(cls.__name__, name) ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - "{}.{} is declared with extra_field() but the {} extra is " + "{}.{} is declared with from_extra() but the {} extra is " "not available for scope {}".format( cls.__name__, name, name, cls.extras_scope ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 7409e7d3..0b4ca647 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -258,7 +258,7 @@ class DatabaseView(View): class DatabaseContext(Context): "The page listing the tables, views and queries in a database, e.g. /fixtures." - template = "database.html" + documented_template = "database.html" database: str = field(metadata={"help": "The name of the database"}) private: bool = field( @@ -341,7 +341,7 @@ class DatabaseContext(Context): class QueryContext(Context): "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." - template = "query.html" + documented_template = "query.html" database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) diff --git a/datasette/views/row.py b/datasette/views/row.py index ef39dce6..129216b9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -30,7 +30,7 @@ from datasette.utils import ( ) from datasette.plugins import pm from datasette.extras import extra_names_from_request, ExtraScope -from . import Context, extra_field +from . import Context, from_extra from .table import ( display_columns_and_rows, _table_page_data, @@ -43,19 +43,19 @@ from .table_extras import RowExtraContext, resolve_row_extras, table_extra_regis class RowContext(Context): "The page showing an individual row, e.g. /fixtures/facetable/1." - template = "row.html" + documented_template = "row.html" extras_scope = ExtraScope.ROW # Fields resolved by registered extras - their documentation comes # from the description on each Extra class in table_extras.py - columns: list = extra_field() - database: str = extra_field() - database_color: str = extra_field() - foreign_key_tables: list = extra_field() - metadata: dict = extra_field() - primary_keys: list = extra_field() - private: bool = extra_field() - table: str = extra_field() + columns: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + foreign_key_tables: list = from_extra() + metadata: dict = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + table: str = from_extra() # Fields added by the view code ok: bool = field( diff --git a/datasette/views/table.py b/datasette/views/table.py index 1e182937..449c6216 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -49,7 +49,7 @@ import sqlite_utils from dataclasses import dataclass, field, fields from datasette.extras import ExtraScope -from . import Context, extra_field +from . import Context, from_extra from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_create_alter import ( @@ -73,40 +73,40 @@ from .table_extras import ( class TableContext(Context): "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." - template = "table.html" + documented_template = "table.html" extras_scope = ExtraScope.TABLE # Fields resolved by registered extras - their documentation comes # from the description on each Extra class in table_extras.py - actions: callable = extra_field() - all_columns: list = extra_field() - columns: list = extra_field() - count: int = extra_field() - count_sql: str = extra_field() - custom_table_templates: list = extra_field() - database: str = extra_field() - database_color: str = extra_field() - display_columns: list = extra_field() - display_rows: list = extra_field() - expandable_columns: list = extra_field() - facet_results: dict = extra_field() - facets_timed_out: list = extra_field() - filters: Filters = extra_field() - form_hidden_args: list = extra_field() - human_description_en: str = extra_field() - is_view: bool = extra_field() - metadata: dict = extra_field() - next_url: str = extra_field() - primary_keys: list = extra_field() - private: bool = extra_field() - query: dict = extra_field() - renderers: dict = extra_field() - set_column_type_ui: dict = extra_field() - sorted_facet_results: list = extra_field() - suggested_facets: list = extra_field() - table: str = extra_field() - table_definition: str = extra_field() - view_definition: str = extra_field() + actions: callable = from_extra() + all_columns: list = from_extra() + columns: list = from_extra() + count: int = from_extra() + count_sql: str = from_extra() + custom_table_templates: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + display_columns: list = from_extra() + display_rows: list = from_extra() + expandable_columns: list = from_extra() + facet_results: dict = from_extra() + facets_timed_out: list = from_extra() + filters: Filters = from_extra() + form_hidden_args: list = from_extra() + human_description_en: str = from_extra() + is_view: bool = from_extra() + metadata: dict = from_extra() + next_url: str = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + query: dict = from_extra() + renderers: dict = from_extra() + set_column_type_ui: dict = from_extra() + sorted_facet_results: list = from_extra() + suggested_facets: list = from_extra() + table: str = from_extra() + table_definition: str = from_extra() + view_definition: str = from_extra() # Fields added by the view code ok: bool = field( diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index c3ec77e2..a5f4fb6f 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -27,7 +27,7 @@ def template_context(cog): for klass in PAGES.values(): title = "{} page".format(klass.__name__.removesuffix("Context")) intro = "{} Rendered using the ``{}`` template.".format( - klass.__doc__, klass.template + klass.__doc__, klass.documented_template ) _section(cog, title, intro) if klass.extras_scope is not None: diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 8d78bf77..4c452774 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -40,20 +40,22 @@ def test_context_class_fields_all_have_help(klass): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) -def test_context_class_has_docstring_and_template(klass): +def test_context_class_has_docstring_and_documented_template(klass): assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) - assert klass.template, "{} is missing a template".format(klass.__name__) + assert klass.documented_template, "{} is missing a documented_template".format( + klass.__name__ + ) -def test_extra_field_documentation_comes_from_the_extra_class(): - from datasette.views import extra_field +def test_from_extra_documentation_comes_from_the_extra_class(): + from datasette.views import from_extra from datasette.views.table_extras import CountExtra @dataclass class DemoContext(Context): extras_scope = ExtraScope.TABLE - count: int = extra_field() + count: int = from_extra() name: str = field(metadata={"help": "The name"}) fields = {f.name: f for f in DemoContext.documented_fields()} @@ -63,28 +65,28 @@ def test_extra_field_documentation_comes_from_the_extra_class(): assert not fields["name"].from_extra -def test_extra_field_must_match_a_registered_extra(): - from datasette.views import extra_field +def test_from_extra_must_match_a_registered_extra(): + from datasette.views import from_extra @dataclass class BadContext(Context): extras_scope = ExtraScope.TABLE - not_a_real_extra: str = extra_field() + not_a_real_extra: str = from_extra() with pytest.raises(KeyError): BadContext.documented_fields() -def test_extra_field_must_be_available_for_the_scope(): - from datasette.views import extra_field +def test_from_extra_must_be_available_for_the_scope(): + from datasette.views import from_extra @dataclass class WrongScopeContext(Context): extras_scope = ExtraScope.ROW # count is a TABLE-scope extra, not available for ROW - count: int = extra_field() + count: int = from_extra() with pytest.raises(ValueError): WrongScopeContext.documented_fields() From 34d9a3bf33e3c4ff610f7d3a633f63c3d7272cc9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:11:26 -0700 Subject: [PATCH 17/20] Use dataclasses for database table context --- datasette/app.py | 17 +++++++++- datasette/views/__init__.py | 17 +++++++++- datasette/views/database.py | 57 +++++++++++++++++++++------------- docs/contributing.rst | 13 ++++++++ docs/template_context.rst | 8 ++--- tests/test_template_context.py | 10 ++++++ 6 files changed, 95 insertions(+), 27 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 90de60a9..57f893fe 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -331,6 +331,15 @@ def _to_string(value): return json.dumps(value, default=str) +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + @pass_context def _legacy_template_csrftoken(context): request = context.get("request") @@ -2390,7 +2399,13 @@ class Datasette: } if request and request.args.get("_context") and self.setting("template_debug"): return "
{}
".format( - escape(json.dumps(template_context, default=repr, indent=4)) + escape( + json.dumps( + template_context, + default=_template_context_json_default, + indent=4, + ) + ) ) return await template.render_async(template_context) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 6fba1c90..ed7e175f 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,5 +1,7 @@ from dataclasses import dataclass import dataclasses +import types +import typing @dataclass(frozen=True) @@ -10,6 +12,19 @@ class ContextField: from_extra: bool = False +def _type_name(type_): + if type_ is type(None): + return "None" + origin = typing.get_origin(type_) + args = typing.get_args(type_) + if origin in (typing.Union, types.UnionType): + return " | ".join(_type_name(arg) for arg in args) + if origin is not None: + name = getattr(origin, "__name__", str(origin).removeprefix("typing.")) + return "{}[{}]".format(name, ", ".join(_type_name(arg) for arg in args)) + return getattr(type_, "__name__", str(type_).removeprefix("typing.")) + + def from_extra(): """ Declare a Context dataclass field whose value comes from a registered @@ -42,7 +57,7 @@ class Context: documented.append( ContextField( name=f.name, - type_name=getattr(f.type, "__name__", str(f.type)), + type_name=_type_name(f.type), help=help_text, from_extra=is_from_extra, ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0b4ca647..fd985b77 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -45,6 +45,21 @@ from .table_create_alter import _create_table_ui_context from . import Context +@dataclass +class DatabaseTable: + "Summary of a table or view shown on database and query pages." + + name: str + columns: list[str] + primary_keys: list[str] + count: int | None + count_truncated: bool + hidden: bool + fts_table: str | None + foreign_keys: dict[str, list[dict[str, str]]] + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -169,8 +184,8 @@ class DatabaseView(View): "private": private, "path": datasette.urls.database(database), "size": db.size, - "tables": tables, - "hidden_count": len([t for t in tables if t["hidden"]]), + "tables": [asdict(table) for table in tables], + "hidden_count": len([table for table in tables if table.hidden]), "views": sql_views, "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, @@ -211,7 +226,7 @@ class DatabaseView(View): path=datasette.urls.database(database), size=db.size, tables=tables, - hidden_count=len([t for t in tables if t["hidden"]]), + hidden_count=len([table for table in tables if table.hidden]), views=sql_views, queries=stored_queries, queries_more=queries_more, @@ -266,9 +281,9 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field( + tables: list[DatabaseTable] = field( metadata={ - "help": "List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." } ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) @@ -391,9 +406,9 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field( + tables: list[DatabaseTable] = field( metadata={ - "help": "List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." } ) named_parameter_values: dict = field( @@ -456,7 +471,7 @@ class QueryContext(Context): ) -async def get_tables(datasette, request, db, allowed_dict): +async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable]: """ Get list of tables with metadata for the database view. @@ -477,21 +492,21 @@ async def get_tables(datasette, request, db, allowed_dict): table_columns = await db.table_columns(table) tables.append( - { - "name": table, - "columns": table_columns, - "primary_keys": await db.primary_keys(table), - "count": table_counts[table], - "count_truncated": _table_count_truncated( + DatabaseTable( + name=table, + columns=table_columns, + primary_keys=await db.primary_keys(table), + count=table_counts[table], + count_truncated=_table_count_truncated( datasette, db, table, table_counts[table] ), - "hidden": table in hidden_table_names, - "fts_table": await db.fts_table(table), - "foreign_keys": all_foreign_keys[table], - "private": allowed_dict[table].private, - } + hidden=table in hidden_table_names, + fts_table=await db.fts_table(table), + foreign_keys=all_foreign_keys[table], + private=allowed_dict[table].private, + ) ) - tables.sort(key=lambda t: (t["hidden"], t["name"])) + tables.sort(key=lambda table: (table.hidden, table.name)) return tables diff --git a/docs/contributing.rst b/docs/contributing.rst index 3b45834c..142c2f41 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -309,6 +309,19 @@ To update these pages, run the following command:: uv run cog -r docs/*.rst +.. _contributing_template_contexts: + +Documented template contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's documented template contexts are part of the public API for custom templates. They are defined as dataclasses next to the view code that renders them, for example ``DatabaseContext`` and ``QueryContext`` in ``datasette/views/database.py``. + +Every documented context class inherits from ``datasette.views.Context``. Fields that are added directly by view code should be declared as dataclass fields with ``help`` metadata, which is used to generate :ref:`template_context`. Fields resolved through the page extras system should use ``from_extra()`` so their documentation comes from the matching ``Extra`` class. + +Use ``documented_template`` on each context class to record the canonical template named in the generated documentation. This should be a string such as ``"database.html"``. Runtime template selection still happens in the view code, since most pages consider more specific template names before falling back to the canonical one. + +When a context field contains repeated structured data, prefer a small nested dataclass over an anonymous dictionary. For example, a field containing table summaries should be annotated as ``list[DatabaseTable]`` where ``DatabaseTable`` is a dataclass describing the keys and value types. This keeps the Python contract and generated documentation clear. JSON responses and ``?_context=1`` debug output will convert nested dataclasses back to JSON objects at the response boundary. + .. _contributing_continuous_deployment: Continuously deployed demo instances diff --git a/docs/template_context.rst b/docs/template_context.rst index 12f656d0..311de8bb 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -145,8 +145,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``table_columns`` - ``dict`` Dictionary mapping table names to lists of column names, used to power SQL autocomplete. -``tables`` - ``list`` - List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. +``tables`` - ``list[DatabaseTable]`` + List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Async callable that renders the ``top_database`` plugin slot for this database and returns HTML. @@ -234,8 +234,8 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie ``table_columns`` - ``dict`` Dictionary mapping table names to lists of column names, used to power SQL autocomplete. -``tables`` - ``list`` - List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. +``tables`` - ``list[DatabaseTable]`` + List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Async callable that renders the ``top_query`` plugin slot for this query and returns HTML. diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4c452774..f1919891 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -18,16 +18,22 @@ from datasette.views import Context def test_documented_fields(): + @dataclass + class DemoNested: + name: str + @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) _internal: str = field() count: int = field(metadata={"help": "How many there are"}) + items: list[DemoNested] = field(metadata={"help": "Nested items"}) fields = DemoContext.documented_fields() assert [(f.name, f.type_name, f.help) for f in fields] == [ ("name", "str", "The name"), ("count", "int", "How many there are"), + ("items", "list[DemoNested]", "Nested items"), ] @@ -230,3 +236,7 @@ def test_template_context_docs_cover_every_documented_key(): assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( context_field.name, page_name ) + assert ( + "``{}`` - ``{}``".format(context_field.name, context_field.type_name) + in docs + ), "{} type ({} page)".format(context_field.name, page_name) From 0d1c097396b8e39f4c5d20ea16282c295d2e4e0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:16:42 -0700 Subject: [PATCH 18/20] Document database views and queries as dataclasses --- datasette/views/database.py | 22 +++++++++++++++------- docs/template_context.rst | 8 ++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index fd985b77..e02de657 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,7 +11,7 @@ import textwrap from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import stored_query_to_dict +from datasette.stored_queries import StoredQuery, stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, @@ -60,6 +60,14 @@ class DatabaseTable: private: bool +@dataclass +class DatabaseViewInfo: + "Summary of a SQLite view shown on the database page." + + name: str + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -109,7 +117,7 @@ class DatabaseView(View): # Filter to just views view_names_set = set(await db.view_names()) sql_views = [ - {"name": name, "private": allowed_dict[name].private} + DatabaseViewInfo(name=name, private=allowed_dict[name].private) for name in allowed_dict if name in view_names_set ] @@ -186,7 +194,7 @@ class DatabaseView(View): "size": db.size, "tables": [asdict(table) for table in tables], "hidden_count": len([table for table in tables if table.hidden]), - "views": sql_views, + "views": [asdict(view) for view in sql_views], "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, @@ -287,14 +295,14 @@ class DatabaseContext(Context): } ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) - views: list = field( + views: list[DatabaseViewInfo] = field( metadata={ - "help": "List of SQLite view dictionaries. Each item has ``name`` and ``private`` keys." + "help": "List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes." } ) - queries: list = field( + queries: list[StoredQuery] = field( metadata={ - "help": "List of stored query objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." + "help": "List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." } ) queries_more: bool = field( diff --git a/docs/template_context.rst b/docs/template_context.rst index 311de8bb..52ccca49 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -124,8 +124,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``private`` - ``bool`` Boolean indicating if this is a private database -``queries`` - ``list`` - List of stored query objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``. +``queries`` - ``list[StoredQuery]`` + List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``. ``queries_count`` - ``int`` Count of visible stored queries @@ -151,8 +151,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``top_database`` - ``callable`` Async callable that renders the ``top_database`` plugin slot for this database and returns HTML. -``views`` - ``list`` - List of SQLite view dictionaries. Each item has ``name`` and ``private`` keys. +``views`` - ``list[DatabaseViewInfo]`` + List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes. Query page ---------- From 8276879997661ae618ffe984ab6a183d3b62788f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:24:42 -0700 Subject: [PATCH 19/20] Construct table context explicitly --- datasette/views/table.py | 110 ++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 36 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 449c6216..4b923d20 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -46,7 +46,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field from datasette.extras import ExtraScope from . import Context, from_extra @@ -1825,44 +1825,82 @@ async def table_view_traced(datasette, request): ) } ) - # Only keys declared on TableContext are part of the documented - # template contract - anything else in data (e.g. extras requested - # with ?_extra= on the HTML page, or extra filter context added by - # filters_from_request plugins) is dropped here - declared_fields = {f.name for f in fields(TableContext)} + table_context = TableContext( + actions=data["actions"], + all_columns=data["all_columns"], + columns=data["columns"], + count=data["count"], + count_sql=data["count_sql"], + custom_table_templates=data["custom_table_templates"], + database=data["database"], + database_color=data["database_color"], + display_columns=data["display_columns"], + display_rows=data["display_rows"], + expandable_columns=data["expandable_columns"], + facet_results=data["facet_results"], + facets_timed_out=data["facets_timed_out"], + filters=data["filters"], + form_hidden_args=data["form_hidden_args"], + human_description_en=data["human_description_en"], + is_view=data["is_view"], + metadata=data["metadata"], + next_url=data["next_url"], + primary_keys=data["primary_keys"], + private=data["private"], + query=data["query"], + renderers=data["renderers"], + set_column_type_ui=data["set_column_type_ui"], + sorted_facet_results=data["sorted_facet_results"], + suggested_facets=data["suggested_facets"], + table=data["table"], + table_definition=data["table_definition"], + view_definition=data["view_definition"], + ok=data["ok"], + next=data["next"], + count_truncated=data["count_truncated"], + rows=data["rows"], + filter_columns=data["filter_columns"], + supports_search=data["supports_search"], + extra_wheres_for_ui=data["extra_wheres_for_ui"], + url_csv=data["url_csv"], + url_csv_path=data["url_csv_path"], + url_csv_hidden_args=data["url_csv_hidden_args"], + sort=data["sort"], + sort_desc=data["sort_desc"], + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.allowed( + action="execute-sql", + resource=DatabaseResource(database=resolved.db.name), + actor=request.actor, + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), + table_page_data=data["table_page_data"], + table_insert_ui=data["table_insert_ui"], + table_alter_ui=data["table_alter_ui"], + ) r = Response.html( await datasette.render_template( template, - TableContext( - **{k: v for k, v in data.items() if k in declared_fields}, - append_querystring=append_querystring, - path_with_replaced_args=path_with_replaced_args, - fix_path=datasette.urls.path, - settings=datasette.settings_dict(), - # TODO: review up all of these hacks: - alternate_url_json=alternate_url_json, - datasette_allow_facet=( - "true" if datasette.setting("allow_facet") else "false" - ), - is_sortable=any(c["sortable"] for c in data["display_columns"]), - allow_execute_sql=await datasette.allowed( - action="execute-sql", - resource=DatabaseResource(database=resolved.db.name), - actor=request.actor, - ), - query_ms=1.2, - select_templates=[ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - top_table=make_slot_function( - "top_table", - datasette, - request, - database=resolved.db.name, - table=resolved.table, - ), - ), + table_context, request=request, view_name="table", ), From 0c523dda209e3b545c52cd3b113d788cba9a342b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:29:21 -0700 Subject: [PATCH 20/20] Remove count truncated context test --- tests/test_template_context.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index f1919891..7923d0e7 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -191,33 +191,6 @@ async def test_template_context_matches_documented_contract( ) -@pytest.mark.asyncio -async def test_count_truncated_replaces_count_limit_context_key(context_ds): - db = context_ds.databases["fixtures"] - previous_count_limit = db.count_limit - previous_cached_table_counts = db._cached_table_counts - db.count_limit = 10 - db._cached_table_counts = None - try: - table_context = await get_template_context(context_ds, "/fixtures/facetable") - assert table_context["count"] == 11 - assert table_context["count_truncated"] is True - assert "count_limit" not in table_context - - database_context = await get_template_context(context_ds, "/fixtures") - facetable = next( - table - for table in database_context["tables"] - if table["name"] == "facetable" - ) - assert facetable["count"] == 11 - assert facetable["count_truncated"] is True - assert "count_limit" not in database_context - finally: - db.count_limit = previous_count_limit - db._cached_table_counts = previous_cached_table_counts - - def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name)