diff --git a/datasette/app.py b/datasette/app.py index 79dffb66..57f893fe 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -35,6 +35,7 @@ from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, + pass_context, PrefixLoader, ) from jinja2.environment import Template @@ -330,6 +331,50 @@ def _to_string(value): return json.dumps(value, default=str) +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", + "show_logout": "True if the logout link should be shown in the navigation menu", + "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", + "edit_tools_js_hash": "Hash of Datasette's edit-tools.js contents, used for cache busting", + "table_js_hash": "Hash of Datasette's table.js contents, used for cache busting", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - """.format(escape(ex.sql))).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except (sqlite3.OperationalError, InvalidSql) as e: - raise DatasetteError(str(e), title="Invalid SQL", status=400) - - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) - - except DatasetteError: - raise - - end = time.perf_counter() - data["query_ms"] = (end - start) * 1000 - - # Special case for .jsono extension - redirect to _shape=objects - if _format == "jsono": - return self.redirect( - request, - path_with_added_args( - request, - {"_shape": "objects"}, - path=request.path.rsplit(".jsono", 1)[0] + ".json", - ), - forward_querystring=False, - ) - - if _format in self.ds.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - self.ds.renderers[_format][0], - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - truncated=False, # TODO: support this - error=data.get("error"), - # These will be deprecated in Datasette 1.0: - args=request.args, - data=data, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code", status_code or 200), - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), - ) - elif isinstance(result, Response): - r = result - if status_code is not None: - # Over-ride the status code - r.status = status_code - else: - assert False, f"{result} should be dict or Response" - else: - extras = {} - if callable(extra_template_data): - extras = extra_template_data() - if asyncio.iscoroutine(extras): - extras = await extras - else: - extras = extra_template_data - url_labels_extra = {} - if data.get("expandable_columns"): - url_labels_extra = {"_labels": "on"} - - renderers = {} - for key, (_, can_render) in self.ds.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - - url_csv_args = {"_size": "max", **url_labels_extra} - url_csv = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format="csv", - extra_qs=url_csv_args, - ) - ) - url_csv_path = url_csv.split("?")[0] - context = { - **data, - **extras, - **{ - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - }, - } - if "metadata" not in context: - context["metadata"] = await self.ds.get_instance_metadata() - r = await self.render(templates, request=request, context=context) - if status_code is not None: - r.status = status_code - - ttl = request.args.get("_ttl", None) - if ttl is None or not ttl.isdigit(): - ttl = self.ds.setting("default_cache_ttl") - - return self.set_response_headers(r, ttl) - - def set_response_headers(self, response, ttl): - # Set far-future cache expiry - if self.ds.cache_headers and response.status == 200: - ttl = int(ttl) - if ttl == 0: - ttl_header = "no-cache" - else: - ttl_header = f"max-age={ttl}" - response.headers["Cache-Control"] = ttl_header - response.headers["Referrer-Policy"] = "no-referrer" - if self.ds.cors: - add_cors_headers(response.headers) - return response - - def _error(messages, status=400): return Response.json({"ok": False, "errors": messages}, status=status) diff --git a/datasette/views/database.py b/datasette/views/database.py index 8e9c1361..e02de657 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -11,7 +11,7 @@ import textwrap from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import stored_query_to_dict +from datasette.stored_queries import StoredQuery, stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, @@ -45,6 +45,29 @@ from .table_create_alter import _create_table_ui_context from . import Context +@dataclass +class DatabaseTable: + "Summary of a table or view shown on database and query pages." + + name: str + columns: list[str] + primary_keys: list[str] + count: int | None + count_truncated: bool + hidden: bool + fts_table: str | None + foreign_keys: dict[str, list[dict[str, str]]] + private: bool + + +@dataclass +class DatabaseViewInfo: + "Summary of a SQLite view shown on the database page." + + name: str + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -94,7 +117,7 @@ class DatabaseView(View): # Filter to just views view_names_set = set(await db.view_names()) sql_views = [ - {"name": name, "private": allowed_dict[name].private} + DatabaseViewInfo(name=name, private=allowed_dict[name].private) for name in allowed_dict if name in view_names_set ] @@ -169,9 +192,9 @@ class DatabaseView(View): "private": private, "path": datasette.urls.database(database), "size": db.size, - "tables": tables, - "hidden_count": len([t for t in tables if t["hidden"]]), - "views": sql_views, + "tables": [asdict(table) for table in tables], + "hidden_count": len([table for table in tables if table.hidden]), + "views": [asdict(view) for view in sql_views], "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, @@ -211,7 +234,7 @@ class DatabaseView(View): path=datasette.urls.database(database), size=db.size, tables=tables, - hidden_count=len([t for t in tables if t["hidden"]]), + hidden_count=len([table for table in tables if table.hidden]), views=sql_views, queries=stored_queries, queries_more=queries_more, @@ -230,7 +253,6 @@ class DatabaseView(View): database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, - count_limit=db.count_limit, allow_download=datasette.setting("allow_download") and not db.is_mutable and not db.is_memory, @@ -257,16 +279,32 @@ class DatabaseView(View): @dataclass class DatabaseContext(Context): + "The page listing the tables, views and queries in a database, e.g. /fixtures." + + documented_template = "database.html" + database: str = field(metadata={"help": "The name of the database"}) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list[DatabaseTable] = field( + metadata={ + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + } + ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) - views: list = field(metadata={"help": "List of view objects in the database"}) - queries: list = field(metadata={"help": "List of stored query objects"}) + views: list[DatabaseViewInfo] = field( + metadata={ + "help": "List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes." + } + ) + queries: list[StoredQuery] = field( + metadata={ + "help": "List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." + } + ) queries_more: bool = field( metadata={"help": "Boolean indicating if more stored queries are available"} ) @@ -275,48 +313,65 @@ class DatabaseContext(Context): metadata={"help": "Boolean indicating if custom SQL can be executed"} ) table_columns: dict = field( - metadata={"help": "Dictionary mapping table names to their column lists"} + metadata={ + "help": "Dictionary mapping table names to lists of column names, used to power SQL autocomplete." + } + ) + metadata: dict = field( + metadata={ + "help": "Metadata dictionary for the database, such as ``title``, ``description``, ``license`` and ``source`` values from Datasette metadata." + } ) - metadata: dict = field(metadata={"help": "Metadata for the database"}) database_color: str = field(metadata={"help": "The color assigned to the database"}) database_page_data: dict = field( - metadata={"help": "JSON data used by JavaScript on the database page"} + metadata={ + "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.' + } ) database_actions: callable = field( metadata={ - "help": "Callable returning list of action links for the database menu" + "help": 'Async callable returning action items for the database menu. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions` and :ref:`plugin_hook_database_actions`.' } ) show_hidden: str = field(metadata={"help": "Value of _show_hidden query parameter"}) editable: bool = field( metadata={"help": "Boolean indicating if the database is editable"} ) - count_limit: int = field(metadata={"help": "The maximum number of rows to count"}) allow_download: bool = field( metadata={"help": "Boolean indicating if database download is allowed"} ) attached_databases: list = field( - metadata={"help": "List of names of attached databases"} + metadata={ + "help": "List of names of databases attached to this SQLite connection. This is only populated for the special ``/_memory`` database when Datasette is started with ``--crossdb`` for :ref:`cross_database_queries`." + } ) alternate_url_json: str = field( metadata={"help": "URL for the alternate JSON version of this page"} ) select_templates: list = field( metadata={ - "help": "List of templates that were considered for rendering this page" + "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``." } ) top_database: callable = field( - metadata={"help": "Callable to render the top_database slot"} + metadata={ + "help": "Async callable that renders the ``top_database`` plugin slot for this database and returns HTML." + } ) @dataclass class QueryContext(Context): + "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." + + documented_template = "query.html" + database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( - metadata={"help": "The SQL query object containing the `sql` string"} + metadata={ + "help": "Dictionary describing the SQL query being executed, with ``sql`` and ``params`` keys." + } ) stored_query: str = field( metadata={"help": "The name of the stored query if this is a stored query"} @@ -333,7 +388,9 @@ class QueryContext(Context): } ) metadata: dict = field( - metadata={"help": "Metadata about the database or the stored query"} + metadata={ + "help": "Metadata dictionary for the database or stored query. Stored query metadata may include options such as ``hide_sql``, ``on_success_message`` and ``on_error_redirect``." + } ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -357,22 +414,44 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list[DatabaseTable] = field( + metadata={ + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + } + ) named_parameter_values: dict = field( - metadata={"help": "Dictionary of parameter names/values"} + metadata={ + "help": "Dictionary of named SQL parameter values, keyed by parameter name without the leading ``:``." + } ) edit_sql_url: str = field( metadata={"help": "URL to edit the SQL for a stored query"} ) - display_rows: list = field(metadata={"help": "List of result rows to display"}) - columns: list = field(metadata={"help": "List of column names"}) - renderers: dict = field(metadata={"help": "Dictionary of renderer name to URL"}) + display_rows: list = field( + metadata={ + "help": "List of result rows formatted for HTML display. Each row is a list of rendered cell values in the same order as ``columns``." + } + ) + columns: list = field( + metadata={ + "help": "List of result column names in the order they appear in ``display_rows`` and ``rows``." + } + ) + renderers: dict = field( + metadata={ + "help": "Dictionary mapping output format names such as ``json`` to URLs for this query in that format." + } + ) url_csv: str = field(metadata={"help": "URL for CSV export"}) show_hide_hidden: str = field( - metadata={"help": "Hidden input field for the _show_sql parameter"} + metadata={ + "help": "Rendered hidden ```` HTML preserving the current ``_hide_sql`` or ``_show_sql`` state." + } ) table_columns: dict = field( - metadata={"help": "Dictionary of table name to list of column names"} + metadata={ + "help": "Dictionary mapping table names to lists of column names, used to power SQL autocomplete." + } ) alternate_url_json: str = field( metadata={"help": "URL for alternate JSON version of this page"} @@ -380,23 +459,27 @@ class QueryContext(Context): # TODO: refactor this to somewhere else, probably ds.render_template() select_templates: list = field( metadata={ - "help": "List of templates that were considered for rendering this page" + "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``." } ) top_query: callable = field( - metadata={"help": "Callable to render the top_query slot"} + metadata={ + "help": "Async callable that renders the ``top_query`` plugin slot for this query and returns HTML." + } ) top_stored_query: callable = field( - metadata={"help": "Callable to render the top_stored_query slot"} + metadata={ + "help": "Async callable that renders the ``top_stored_query`` plugin slot for stored queries and returns HTML." + } ) query_actions: callable = field( metadata={ - "help": "Callable returning a list of links for the query action menu" + "help": 'Async callable returning action items for the query menu. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions` and :ref:`plugin_hook_query_actions`.' } ) -async def get_tables(datasette, request, db, allowed_dict): +async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable]: """ Get list of tables with metadata for the database view. @@ -417,21 +500,36 @@ async def get_tables(datasette, request, db, allowed_dict): table_columns = await db.table_columns(table) tables.append( - { - "name": table, - "columns": table_columns, - "primary_keys": await db.primary_keys(table), - "count": table_counts[table], - "hidden": table in hidden_table_names, - "fts_table": await db.fts_table(table), - "foreign_keys": all_foreign_keys[table], - "private": allowed_dict[table].private, - } + DatabaseTable( + name=table, + columns=table_columns, + primary_keys=await db.primary_keys(table), + count=table_counts[table], + count_truncated=_table_count_truncated( + datasette, db, table, table_counts[table] + ), + hidden=table in hidden_table_names, + fts_table=await db.fts_table(table), + foreign_keys=all_foreign_keys[table], + private=allowed_dict[table].private, + ) ) - tables.sort(key=lambda t: (t["hidden"], t["name"])) + tables.sort(key=lambda table: (table.hidden, table.name)) return tables +def _table_count_truncated(datasette, db, table, count): + if count != db.count_limit + 1: + return False + if not db.is_mutable and datasette.inspect_data: + try: + datasette.inspect_data[db.name]["tables"][table]["count"] + return False + except KeyError: + pass + return True + + async def database_download(request, datasette): from datasette.resources import DatabaseResource diff --git a/datasette/views/row.py b/datasette/views/row.py index e3575b6c..129216b9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,21 +1,36 @@ +import asyncio +import json +import textwrap +import time +import urllib.parse +from dataclasses import dataclass, field + +import markupsafe +import sqlite_utils + from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource -from .base import DataView, BaseView, _error +from .base import BaseView, DatasetteError, _error, stream_csv from datasette.utils import ( + add_cors_headers, await_me_maybe, + call_with_supported_arguments, CustomRow, + InvalidSql, make_slot_function, path_from_row_pks, + path_with_added_args, + path_with_format, + path_with_removed_args, to_css_class, escape_sqlite, + sqlite3, ) from datasette.plugins import pm -import json -import markupsafe -import sqlite_utils -from datasette.extras import extra_names_from_request +from datasette.extras import extra_names_from_request, ExtraScope +from . import Context, from_extra from .table import ( display_columns_and_rows, _table_page_data, @@ -24,9 +39,366 @@ from .table import ( from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry -class RowView(DataView): +@dataclass +class RowContext(Context): + "The page showing an individual row, e.g. /fixtures/facetable/1." + + documented_template = "row.html" + extras_scope = ExtraScope.ROW + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + columns: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + foreign_key_tables: list = from_extra() + metadata: dict = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + table: str = from_extra() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + rows: list = field( + metadata={ + "help": "A single-item list containing this row as a dictionary mapping column name to raw value." + } + ) + primary_key_values: list = field( + metadata={"help": "Values of the primary keys for this row, from the URL"} + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + display_columns: list = field( + metadata={ + "help": "Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys." + } + ) + display_rows: list = field( + metadata={ + "help": "Rows formatted for the HTML table display. Each row is iterable and contains cell dictionaries with ``column``, ``value``, ``raw`` and ``value_type`` keys." + } + ) + custom_table_templates: list = field( + metadata={ + "help": "Custom template names that were considered for displaying this row's table, in lookup order." + } + ) + row_actions: list = field( + metadata={ + "help": 'Row actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions` and :ref:`plugin_hook_row_actions`.' + } + ) + row_mutation_ui: bool = field( + metadata={"help": "True if the row edit/delete JavaScript UI should be enabled"} + ) + table_page_data: dict = field( + metadata={ + "help": "JSON data used by JavaScript on the row page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs." + } + ) + top_row: callable = field( + metadata={ + "help": "Async callable that renders the ``top_row`` plugin slot for this row and returns HTML." + } + ) + renderers: dict = field( + metadata={ + "help": "Dictionary mapping output format names such as ``json`` to URLs for this row in that format." + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "List of ``(name, value)`` pairs for hidden form fields used by the CSV export form, preserving current options while forcing ``_size=max``." + } + ) + settings: dict = field( + metadata={ + "help": "Dictionary of Datasette's current settings, keyed by setting name." + } + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``." + } + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + + +class RowView(BaseView): name = "row" + def redirect(self, request, path, forward_querystring=True, remove_args=None): + if request.query_string and "?" not in path and forward_querystring: + path = f"{path}?{request.query_string}" + if remove_args: + path = path_with_removed_args(request, remove_args, path=path) + response = Response.redirect(path) + response.headers["Link"] = f"<{path}>; rel=preload" + if self.ds.cors: + add_cors_headers(response.headers) + return response + + async def as_csv(self, request, database): + return await stream_csv(self.ds, self.data, request, database) + + async def get(self, request): + db = await self.ds.resolve_database(request) + database = db.name + database_route = db.route + format_ = request.url_vars.get("format") or "html" + data_kwargs = {} + + if format_ == "csv": + return await self.as_csv(request, database_route) + + if format_ == "html": + # HTML views default to expanding all foreign key labels + data_kwargs["default_labels"] = True + + extra_template_data = {} + start = time.perf_counter() + status_code = None + templates = () + try: + response_or_template_contexts = await self.data(request, **data_kwargs) + if isinstance(response_or_template_contexts, Response): + return response_or_template_contexts + # If it has four items, it includes an HTTP status code + if len(response_or_template_contexts) == 4: + ( + data, + extra_template_data, + templates, + status_code, + ) = response_or_template_contexts + else: + data, extra_template_data, templates = response_or_template_contexts + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent(""" +

SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

+ + + """.format(markupsafe.escape(ex.sql))).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except (sqlite3.OperationalError, InvalidSql) as e: + raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: + raise + + end = time.perf_counter() + data["query_ms"] = (end - start) * 1000 + + # Special case for .jsono extension - redirect to _shape=objects + if format_ == "jsono": + return self.redirect( + request, + path_with_added_args( + request, + {"_shape": "objects"}, + path=request.path.rsplit(".jsono", 1)[0] + ".json", + ), + forward_querystring=False, + ) + + if format_ in self.ds.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + self.ds.renderers[format_][0], + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + truncated=False, # TODO: support this + error=data.get("error"), + # These will be deprecated in Datasette 1.0: + args=request.args, + data=data, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + response = Response( + body=result.get("body"), + status=result.get("status_code", status_code or 200), + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + response = result + if status_code is not None: + # Over-ride the status code + response.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + response = await self.html(request, data, extra_template_data, templates) + if status_code is not None: + response.status = status_code + else: + raise NotFound("Invalid format: {}".format(format_)) + + ttl = request.args.get("_ttl", None) + if ttl is None or not ttl.isdigit(): + ttl = self.ds.setting("default_cache_ttl") + + return self.set_response_headers(response, ttl) + + async def html(self, request, data, extra_template_data, templates): + extras = {} + if callable(extra_template_data): + extras = extra_template_data() + if asyncio.iscoroutine(extras): + extras = await extras + else: + extras = extra_template_data + + url_labels_extra = {} + if data.get("expandable_columns"): + url_labels_extra = {"_labels": "on"} + + renderers = {} + for key, (_, can_render) in self.ds.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=data.get("database"), + table=data.get("table"), + request=request, + view_name=self.name, + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + + url_csv_args = {"_size": "max", **url_labels_extra} + url_csv = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="csv", + extra_qs=url_csv_args, + ) + ) + url_csv_path = url_csv.split("?")[0] + context = {**data, **extras} + if "metadata" not in context: + context["metadata"] = await self.ds.get_instance_metadata() + + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="json", + ) + ), + ) + return Response.html( + await self.ds.render_template( + template, + RowContext( + columns=context["columns"], + database=context["database"], + database_color=context["database_color"], + foreign_key_tables=context["foreign_key_tables"], + metadata=context["metadata"], + primary_keys=context["primary_keys"], + private=context["private"], + table=context["table"], + ok=context["ok"], + rows=context["rows"], + primary_key_values=context["primary_key_values"], + query_ms=context["query_ms"], + display_columns=context["display_columns"], + display_rows=context["display_rows"], + custom_table_templates=context["custom_table_templates"], + row_actions=context["row_actions"], + row_mutation_ui=context["row_mutation_ui"], + table_page_data=context["table_page_data"], + top_row=context["top_row"], + renderers=renderers, + url_csv=url_csv, + url_csv_path=url_csv_path, + url_csv_hidden_args=[ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + settings=self.ds.settings_dict(), + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + alternate_url_json=alternate_url_json, + ), + request=request, + view_name=self.name, + ), + headers={ + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, + ) + + def set_response_headers(self, response, ttl): + # Set far-future cache expiry + if self.ds.cache_headers and response.status == 200: + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" + else: + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header + response.headers["Referrer-Policy"] = "no-referrer" + if self.ds.cors: + add_cors_headers(response.headers) + return response + async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) db = resolved.db diff --git a/datasette/views/table.py b/datasette/views/table.py index 855c62cd..4b923d20 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -46,6 +46,10 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils +from dataclasses import dataclass, field + +from datasette.extras import ExtraScope +from . import Context, from_extra from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_create_alter import ( @@ -64,6 +68,153 @@ from .table_extras import ( table_extra_registry, ) + +@dataclass +class TableContext(Context): + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." + + documented_template = "table.html" + extras_scope = ExtraScope.TABLE + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + actions: callable = from_extra() + all_columns: list = from_extra() + columns: list = from_extra() + count: int = from_extra() + count_sql: str = from_extra() + custom_table_templates: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + display_columns: list = from_extra() + display_rows: list = from_extra() + expandable_columns: list = from_extra() + facet_results: dict = from_extra() + facets_timed_out: list = from_extra() + filters: Filters = from_extra() + form_hidden_args: list = from_extra() + human_description_en: str = from_extra() + is_view: bool = from_extra() + metadata: dict = from_extra() + next_url: str = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + query: dict = from_extra() + renderers: dict = from_extra() + set_column_type_ui: dict = from_extra() + sorted_facet_results: list = from_extra() + suggested_facets: list = from_extra() + table: str = from_extra() + table_definition: str = from_extra() + view_definition: str = from_extra() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + count_truncated: bool = field( + metadata={ + "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." + } + ) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to raw value." + } + ) + filter_columns: list = field( + metadata={ + "help": "List of column names offered by the filter interface, including currently displayed columns and any hidden columns that can still be filtered." + } + ) + supports_search: bool = field( + metadata={"help": "True if this table has full-text search configured"} + ) + extra_wheres_for_ui: list = field( + metadata={ + "help": "Extra where clauses from ``?_where=`` for display in the UI. Each item has ``text`` for the SQL fragment and ``remove_url`` for a URL that removes that fragment." + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "List of ``(name, value)`` pairs for hidden form fields used by the CSV export form, preserving current filters while forcing ``_size=max``." + } + ) + sort: str = field(metadata={"help": "Column the page is sorted by, or None"}) + sort_desc: str = field( + metadata={"help": "Column the page is sorted by in descending order, or None"} + ) + append_querystring: callable = field( + metadata={ + "help": "Function ``append_querystring(url, querystring)`` that appends additional query string arguments to a URL, using ``?`` or ``&`` as appropriate." + } + ) + path_with_replaced_args: callable = field( + metadata={ + "help": "Function for building the current path with modified query string arguments. Pass the current ``request`` and a dictionary of argument names to replacement values, using ``None`` to remove an argument." + } + ) + fix_path: callable = field( + metadata={ + "help": "Function that applies the configured ``base_url`` prefix to a path." + } + ) + settings: dict = field( + metadata={ + "help": "Dictionary of Datasette's current settings, keyed by setting name." + } + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + datasette_allow_facet: str = field( + metadata={ + "help": 'The string "true" or "false" reflecting the allow_facet setting' + } + ) + is_sortable: bool = field( + metadata={"help": "True if any of the displayed columns can be used to sort"} + ) + allow_execute_sql: bool = field( + metadata={ + "help": "True if the current actor can execute custom SQL against this database" + } + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``." + } + ) + top_table: callable = field( + metadata={ + "help": "Async callable that renders the ``top_table`` plugin slot for this table or view and returns HTML." + } + ) + table_page_data: dict = field( + metadata={ + "help": "JSON data used by JavaScript on the table page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs, optional ``insertRow`` data and optional ``alterTable`` data." + } + ) + table_insert_ui: dict = field( + metadata={ + "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys." + } + ) + table_alter_ui: dict = field( + metadata={ + "help": "Information needed to enable the alter table UI, or ``None`` if altering this table is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``primaryKeys``, ``columnTypes``, ``defaultExpressions`` and ``foreignKeyTargetsPath`` keys, plus optional ``customColumnTypes`` and ``dropPath`` keys." + } + ) + + LINK_WITH_LABEL = ( '{label} {id}' ) @@ -1260,7 +1411,6 @@ class TableFragmentView(BaseView): path_with_replaced_args=path_with_replaced_args, fix_path=self.ds.urls.path, settings=self.ds.settings_dict(), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1675,40 +1825,82 @@ async def table_view_traced(datasette, request): ) } ) + table_context = TableContext( + actions=data["actions"], + all_columns=data["all_columns"], + columns=data["columns"], + count=data["count"], + count_sql=data["count_sql"], + custom_table_templates=data["custom_table_templates"], + database=data["database"], + database_color=data["database_color"], + display_columns=data["display_columns"], + display_rows=data["display_rows"], + expandable_columns=data["expandable_columns"], + facet_results=data["facet_results"], + facets_timed_out=data["facets_timed_out"], + filters=data["filters"], + form_hidden_args=data["form_hidden_args"], + human_description_en=data["human_description_en"], + is_view=data["is_view"], + metadata=data["metadata"], + next_url=data["next_url"], + primary_keys=data["primary_keys"], + private=data["private"], + query=data["query"], + renderers=data["renderers"], + set_column_type_ui=data["set_column_type_ui"], + sorted_facet_results=data["sorted_facet_results"], + suggested_facets=data["suggested_facets"], + table=data["table"], + table_definition=data["table_definition"], + view_definition=data["view_definition"], + ok=data["ok"], + next=data["next"], + count_truncated=data["count_truncated"], + rows=data["rows"], + filter_columns=data["filter_columns"], + supports_search=data["supports_search"], + extra_wheres_for_ui=data["extra_wheres_for_ui"], + url_csv=data["url_csv"], + url_csv_path=data["url_csv_path"], + url_csv_hidden_args=data["url_csv_hidden_args"], + sort=data["sort"], + sort_desc=data["sort_desc"], + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.allowed( + action="execute-sql", + resource=DatabaseResource(database=resolved.db.name), + actor=request.actor, + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), + table_page_data=data["table_page_data"], + table_insert_ui=data["table_insert_ui"], + table_alter_ui=data["table_alter_ui"], + ) r = Response.html( await datasette.render_template( template, - dict( - data, - append_querystring=append_querystring, - path_with_replaced_args=path_with_replaced_args, - fix_path=datasette.urls.path, - settings=datasette.settings_dict(), - # TODO: review up all of these hacks: - alternate_url_json=alternate_url_json, - datasette_allow_facet=( - "true" if datasette.setting("allow_facet") else "false" - ), - is_sortable=any(c["sortable"] for c in data["display_columns"]), - allow_execute_sql=await datasette.allowed( - action="execute-sql", - resource=DatabaseResource(database=resolved.db.name), - actor=request.actor, - ), - query_ms=1.2, - select_templates=[ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - top_table=make_slot_function( - "top_table", - datasette, - request, - database=resolved.db.name, - table=resolved.table, - ), - count_limit=resolved.db.count_limit, - ), + table_context, request=request, view_name="table", ), @@ -2140,6 +2332,9 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: + data["count_truncated"] = _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, data.get("count") + ) data.update(extra_context_from_filters) # filter_columns combine the columns we know are available # in the table with any additional columns (such as rowid) @@ -2205,6 +2400,24 @@ async def table_view_data( return data, rows[:page_size], columns, expanded_columns, sql, next_url +def _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, count +): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 7cb4d8f0..db659c80 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -98,7 +98,7 @@ class QueryExtraContext: class CountSqlExtra(Extra): - description = "SQL query used to calculate the total count" + description = "SQL query string used to calculate the total count for the current table view, including active filters." example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") scopes = {ExtraScope.TABLE} @@ -127,8 +127,8 @@ class CountExtra(Extra): pass if context.count_sql and count is None and not context.nocount: - count_sql_limited = ( - f"select count(*) from (select * {context.from_sql} limit 10001)" + count_sql_limited = "select count(*) from (select * {} limit {})".format( + context.from_sql, context.db.count_limit + 1 ) try: count_rows = list( @@ -165,7 +165,7 @@ class FacetInstancesProvider(Provider): class FacetResultsExtra(Extra): - description = "Results of facets calculated against this data" + description = "Results of facets calculated against this data. A dictionary with ``results`` and ``timed_out`` keys: ``results`` maps facet names to facet dictionaries with ``name``, ``type``, ``results`` and URL keys, and each facet result item includes ``value``, ``label``, ``count`` and ``toggle_url``." example = ExtraExample( value={ "results": { @@ -214,7 +214,9 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): - description = "Facet calculations that timed out" + description = ( + "List of names of facet calculations that exceeded the facet time limit." + ) example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out", note=( @@ -230,7 +232,7 @@ class FacetsTimedOutExtra(Extra): class SuggestedFacetsExtra(Extra): - description = "Suggestions for facets that might return interesting results" + description = "Suggestions for facets that might return interesting results. Each item is a dictionary with ``name`` and ``toggle_url`` keys, and may include extra keys such as ``type`` or ``label`` depending on the facet class." example = ExtraExample( value=[ { @@ -301,7 +303,7 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): - description = "Column names returned by this query" + description = "List of column names returned by this table, row or query." example = ExtraExample("/fixtures/facetable.json?_extra=columns") examples = { ExtraScope.ROW: ExtraExample( @@ -318,7 +320,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): - description = "All columns in the table, regardless of _col/_nocol filtering" + description = "List of all column names in the table, regardless of ``_col=`` or ``_nocol=`` filtering." example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") scopes = {ExtraScope.TABLE} @@ -327,7 +329,7 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): - description = "Primary keys for this table" + description = "List of primary key column names for this table, or an empty list if the table has no explicit primary key." example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") examples = { ExtraScope.ROW: ExtraExample( @@ -341,7 +343,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): - description = "Table or view actions made available by plugin hooks" + description = 'Async callable returning table or view actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions`, :ref:`plugin_hook_table_actions` and :ref:`plugin_hook_view_actions`.' scopes = {ExtraScope.TABLE} # Returns an async function for the HTML templates - not JSON serializable public = False @@ -421,7 +423,7 @@ class IsViewExtra(Extra): class DebugExtra(Extra): - description = "Extra debug information" + description = "Extra debug information dictionary. This is intended for development only and its shape is not part of the stable template contract." docs_note = ( "The contents of this block are not a stable part of the Datasette " "API and may change without warning." @@ -457,7 +459,7 @@ class DebugExtra(Extra): class RequestExtra(Extra): - description = "Full information about the request" + description = "Dictionary with request details: ``url``, ``path``, ``full_path``, ``host`` and ``args`` where ``args`` maps query string parameter names to their values." example = ExtraExample("/fixtures/facetable.json?_extra=request") examples = { ExtraScope.ROW: ExtraExample( @@ -501,7 +503,7 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): - description = "Column metadata used by the HTML table display" + description = "Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys." example = ExtraExample( value=[ { @@ -531,7 +533,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): - description = "Row data formatted for the HTML table display" + description = "Rows formatted for the HTML table display. Each row is iterable and contains cell dictionaries with ``column``, ``value``, ``raw`` and ``value_type`` keys; table pages may also provide ``pk_path``, ``row_path`` and ``row_label`` attributes on each row object." scopes = {ExtraScope.TABLE} # Contains markupsafe/sqlite3.Row values - not JSON serializable public = False @@ -640,7 +642,7 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): - description = "Details of the underlying SQL query" + description = "Details of the underlying SQL query as a dictionary with ``sql`` and ``params`` keys." example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") examples = { ExtraScope.ROW: ExtraExample( @@ -661,7 +663,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): - description = "Column type assignments for this table" + description = 'Column type assignments for this table. A dictionary mapping column names to ``{"type": type_name, "config": config}`` dictionaries.' docs_note = ( "An empty object if no column types have been assigned. Column types " "can be assigned in :ref:`configuration " @@ -700,7 +702,7 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): - description = "Information needed to build an interface for assigning column types" + description = "Information needed to build an interface for assigning column types, or ``None`` if unavailable. When present it has ``path`` and ``columns`` keys; ``columns`` maps column names to ``current`` and ``options`` values." docs_note = ( "``null`` unless the current actor is allowed to use the :ref:`set " "column type API ` for this table." @@ -784,7 +786,7 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): - description = "Metadata about the table, database or stored query" + description = "Metadata dictionary for the table, database or stored query. Table and row metadata include a ``columns`` dictionary mapping column names to descriptions; stored query metadata returns the stored query configuration." docs_note = "See :ref:`metadata` for how to attach metadata to tables." example = ExtraExample( "/fixtures/facetable.json?_extra=metadata", @@ -891,7 +893,7 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): - description = "Hidden form arguments used by the HTML table interface" + description = "List of ``(name, value)`` pairs for hidden form fields used by the HTML table interface to preserve current query string options." example = ExtraExample( "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" ) @@ -911,7 +913,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): - description = "Filters object used by the HTML table interface" + description = "``Filters`` object used by the HTML table interface. Useful methods include ``filters.human_description_en()``; this is not JSON serializable." scopes = {ExtraScope.TABLE} # Returns a Filters instance for the HTML templates - not JSON serializable public = False @@ -921,7 +923,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): - description = "Custom template names considered for this table" + description = "List of custom template names considered for rendering table rows, in lookup order." docs_note = ( "The first template in this list that exists will be used to render " "the table on the HTML version of this page. See " @@ -939,7 +941,7 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): - description = "Facet results sorted for display" + description = "Facet result dictionaries sorted for display. Each item has the same shape as an entry from ``facet_results['results']``." docs_note = ( "The same data as ``facet_results``, as a list in the order used by " "the HTML interface: facets from :ref:`facet configuration " @@ -1001,7 +1003,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): - description = "Alternative output renderers available for this table" + description = "Dictionary mapping output format names such as ``json`` or plugin-provided renderer names to URLs for this data in that format." example = ExtraExample( "/fixtures/facetable.json?_extra=renderers", note=( @@ -1068,7 +1070,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): - description = "Foreign key columns that can be expanded with labels" + description = "List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_column)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_column`` is the label column in the referenced table, or ``None``." docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels." example = ExtraExample( "/fixtures/facetable.json?_extra=expandable_columns", @@ -1090,7 +1092,7 @@ class ExpandableColumnsExtra(Extra): class ForeignKeyTablesExtra(Extra): - description = "Tables that link to this row using foreign keys" + description = "List of tables that link to this row using foreign keys. Each item includes the foreign key fields plus ``count`` for matching rows and ``link`` for the filtered table URL." example = ExtraExample( "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables", note=( @@ -1108,7 +1110,7 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): - description = "List of ?_extra= blocks that can be used on this page" + description = "List of ``?_extra=`` blocks that can be used on this page. Each item has ``name``, ``description``, ``toggle_url`` and ``selected`` keys." example = ExtraExample( value=[ { diff --git a/docs/contributing.rst b/docs/contributing.rst index 3b45834c..142c2f41 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -309,6 +309,19 @@ To update these pages, run the following command:: uv run cog -r docs/*.rst +.. _contributing_template_contexts: + +Documented template contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's documented template contexts are part of the public API for custom templates. They are defined as dataclasses next to the view code that renders them, for example ``DatabaseContext`` and ``QueryContext`` in ``datasette/views/database.py``. + +Every documented context class inherits from ``datasette.views.Context``. Fields that are added directly by view code should be declared as dataclass fields with ``help`` metadata, which is used to generate :ref:`template_context`. Fields resolved through the page extras system should use ``from_extra()`` so their documentation comes from the matching ``Extra`` class. + +Use ``documented_template`` on each context class to record the canonical template named in the generated documentation. This should be a string such as ``"database.html"``. Runtime template selection still happens in the view code, since most pages consider more specific template names before falling back to the canonical one. + +When a context field contains repeated structured data, prefer a small nested dataclass over an anonymous dictionary. For example, a field containing table summaries should be annotated as ``list[DatabaseTable]`` where ``DatabaseTable`` is a dataclass describing the keys and value types. This keeps the Python contract and generated documentation clear. JSON responses and ``?_context=1`` debug output will convert nested dataclasses back to JSON objects at the response boundary. + .. _contributing_continuous_deployment: Continuously deployed demo instances diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 8851a841..f28f1189 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -177,6 +177,11 @@ this:: Datasette will now first look for templates in that directory, and fall back on the defaults if no matches are found. +The variables made available to each template are documented on the +:ref:`template_context` page. Variables documented there are a stable API: +custom templates that use them will keep working in future Datasette +releases, up until the next major version. + It is also possible to over-ride templates on a per-database, per-row or per- table basis. diff --git a/docs/index.rst b/docs/index.rst index c76969bc..d494fd17 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ Contents settings introspection custom_templates + template_context plugins writing_plugins javascript_plugins diff --git a/docs/json_api.rst b/docs/json_api.rst index b8632bc3..eca22fdc 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -267,7 +267,7 @@ The available table extras are listed below. 15 ``count_sql`` - SQL query used to calculate the total count + SQL query string used to calculate the total count for the current table view, including active filters. ``GET /fixtures/facetable.json?_size=0&_extra=count_sql`` @@ -276,7 +276,7 @@ The available table extras are listed below. "select count(*) from facetable " ``facet_results`` - Results of facets calculated against this data (May execute additional queries. See :ref:`facets` for details of how facets work.) + Results of facets calculated against this data. A dictionary with ``results`` and ``timed_out`` keys: ``results`` maps facet names to facet dictionaries with ``name``, ``type``, ``results`` and URL keys, and each facet result item includes ``value``, ``label``, ``count`` and ``toggle_url``. (May execute additional queries. See :ref:`facets` for details of how facets work.) Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. @@ -305,7 +305,7 @@ The available table extras are listed below. } ``facets_timed_out`` - Facet calculations that timed out + List of names of facet calculations that exceeded the facet time limit. ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` @@ -316,7 +316,7 @@ The available table extras are listed below. [] ``suggested_facets`` - Suggestions for facets that might return interesting results (May execute additional queries. Suggestions are controlled by the :ref:`setting_suggest_facets` setting.) + Suggestions for facets that might return interesting results. Each item is a dictionary with ``name`` and ``toggle_url`` keys, and may include extra keys such as ``type`` or ``label`` depending on the facet class. (May execute additional queries. Suggestions are controlled by the :ref:`setting_suggest_facets` setting.) Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. @@ -350,7 +350,7 @@ The available table extras are listed below. "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" ``columns`` - Column names returned by this query + List of column names returned by this table, row or query. ``GET /fixtures/facetable.json?_extra=columns`` @@ -371,7 +371,7 @@ The available table extras are listed below. ] ``all_columns`` - All columns in the table, regardless of _col/_nocol filtering + List of all column names in the table, regardless of ``_col=`` or ``_nocol=`` filtering. ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns`` @@ -392,7 +392,7 @@ The available table extras are listed below. ] ``primary_keys`` - Primary keys for this table + List of primary key column names for this table, or an empty list if the table has no explicit primary key. ``GET /fixtures/facetable.json?_extra=primary_keys`` @@ -403,7 +403,7 @@ The available table extras are listed below. ] ``display_columns`` - Column metadata used by the HTML table display + Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys. Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns. @@ -456,7 +456,7 @@ The available table extras are listed below. } ``debug`` - Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) + Extra debug information dictionary. This is intended for development only and its shape is not part of the stable template contract. (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/facetable.json?_extra=debug`` @@ -474,7 +474,7 @@ The available table extras are listed below. } ``request`` - Full information about the request + Dictionary with request details: ``url``, ``path``, ``full_path``, ``host`` and ``args`` where ``args`` maps query string parameter names to their values. ``GET /fixtures/facetable.json?_extra=request`` @@ -493,7 +493,7 @@ The available table extras are listed below. } ``query`` - Details of the underlying SQL query + Details of the underlying SQL query as a dictionary with ``sql`` and ``params`` keys. ``GET /fixtures/facetable.json?_size=1&_extra=query`` @@ -505,7 +505,7 @@ The available table extras are listed below. } ``column_types`` - Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) + Column type assignments for this table. A dictionary mapping column names to ``{"type": type_name, "config": config}`` dictionaries. (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) ``GET /fixtures/facetable.json?_size=0&_extra=column_types`` @@ -521,7 +521,7 @@ The available table extras are listed below. } ``set_column_type_ui`` - Information needed to build an interface for assigning column types (``null`` unless the current actor is allowed to use the :ref:`set column type API ` for this table.) + Information needed to build an interface for assigning column types, or ``None`` if unavailable. When present it has ``path`` and ``columns`` keys; ``columns`` maps column names to ``current`` and ``options`` values. (``null`` unless the current actor is allowed to use the :ref:`set column type API ` for this table.) Shape abbreviated to two columns, as seen by an actor with ``set-column-type`` permission. ``current`` is the column type currently assigned to each column and ``options`` lists the types that could be assigned to it. @@ -571,7 +571,7 @@ The available table extras are listed below. } ``metadata`` - Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + Metadata dictionary for the table, database or stored query. Table and row metadata include a ``columns`` dictionary mapping column names to descriptions; stored query metadata returns the stored query configuration. (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/facetable.json?_extra=metadata`` @@ -587,7 +587,7 @@ The available table extras are listed below. } ``extras`` - List of ?_extra= blocks that can be used on this page + List of ``?_extra=`` blocks that can be used on this page. Each item has ``name``, ``description``, ``toggle_url`` and ``selected`` keys. Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. @@ -636,7 +636,7 @@ The available table extras are listed below. "9403e5" ``renderers`` - Alternative output renderers available for this table + Dictionary mapping output format names such as ``json`` or plugin-provided renderer names to URLs for this data in that format. ``GET /fixtures/facetable.json?_extra=renderers`` @@ -649,7 +649,7 @@ The available table extras are listed below. } ``custom_table_templates`` - Custom template names considered for this table (The first template in this list that exists will be used to render the table on the HTML version of this page. See :ref:`customization_custom_templates`.) + List of custom template names considered for rendering table rows, in lookup order. (The first template in this list that exists will be used to render the table on the HTML version of this page. See :ref:`customization_custom_templates`.) ``GET /fixtures/facetable.json?_extra=custom_table_templates`` @@ -662,7 +662,7 @@ The available table extras are listed below. ] ``sorted_facet_results`` - Facet results sorted for display (The same data as ``facet_results``, as a list in the order used by the HTML interface: facets from :ref:`facet configuration ` first, then other facets ordered by their number of results.) + Facet result dictionaries sorted for display. Each item has the same shape as an entry from ``facet_results['results']``. (The same data as ``facet_results``, as a list in the order used by the HTML interface: facets from :ref:`facet configuration ` first, then other facets ordered by their number of results.) ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` @@ -738,7 +738,7 @@ The available table extras are listed below. false ``expandable_columns`` - Foreign key columns that can be expanded with labels (See :ref:`expand_foreign_keys` for how to expand these labels.) + List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_column)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_column`` is the label column in the referenced table, or ``None``. (See :ref:`expand_foreign_keys` for how to expand these labels.) ``GET /fixtures/facetable.json?_extra=expandable_columns`` @@ -758,7 +758,7 @@ The available table extras are listed below. ] ``form_hidden_args`` - Hidden form arguments used by the HTML table interface + List of ``(name, value)`` pairs for hidden form fields used by the HTML table interface to preserve current query string options. ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args`` @@ -785,7 +785,7 @@ Row JSON responses The following extras are available for row JSON responses. ``columns`` - Column names returned by this query + List of column names returned by this table, row or query. ``GET /fixtures/simple_primary_key/1.json?_extra=columns`` @@ -797,7 +797,7 @@ The following extras are available for row JSON responses. ] ``primary_keys`` - Primary keys for this table + List of primary key column names for this table, or an empty list if the table has no explicit primary key. ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys`` @@ -829,7 +829,7 @@ The following extras are available for row JSON responses. } ``debug`` - Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) + Extra debug information dictionary. This is intended for development only and its shape is not part of the stable template contract. (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` @@ -858,7 +858,7 @@ The following extras are available for row JSON responses. } ``request`` - Full information about the request + Dictionary with request details: ``url``, ``path``, ``full_path``, ``host`` and ``args`` where ``args`` maps query string parameter names to their values. ``GET /fixtures/simple_primary_key/1.json?_extra=request`` @@ -877,7 +877,7 @@ The following extras are available for row JSON responses. } ``query`` - Details of the underlying SQL query + Details of the underlying SQL query as a dictionary with ``sql`` and ``params`` keys. ``GET /fixtures/simple_primary_key/1.json?_extra=query`` @@ -891,7 +891,7 @@ The following extras are available for row JSON responses. } ``column_types`` - Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) + Column type assignments for this table. A dictionary mapping column names to ``{"type": type_name, "config": config}`` dictionaries. (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) ``GET /fixtures/facetable/1.json?_extra=column_types`` @@ -907,7 +907,7 @@ The following extras are available for row JSON responses. } ``metadata`` - Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + Metadata dictionary for the table, database or stored query. Table and row metadata include a ``columns`` dictionary mapping column names to descriptions; stored query metadata returns the stored query configuration. (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` @@ -920,7 +920,7 @@ The following extras are available for row JSON responses. } ``extras`` - List of ?_extra= blocks that can be used on this page + List of ``?_extra=`` blocks that can be used on this page. Each item has ``name``, ``description``, ``toggle_url`` and ``selected`` keys. Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. @@ -978,7 +978,7 @@ The following extras are available for row JSON responses. false ``foreign_key_tables`` - Tables that link to this row using foreign keys (May execute additional queries.) + List of tables that link to this row using foreign keys. Each item includes the foreign key fields plus ``count`` for matching rows and ``link`` for the filtered table URL. (May execute additional queries.) ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` @@ -1030,7 +1030,7 @@ Query JSON responses The following extras are available for arbitrary SQL query responses and stored, named query responses. ``columns`` - Column names returned by this query + List of column names returned by this table, row or query. ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns`` @@ -1061,7 +1061,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``debug`` - Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) + Extra debug information dictionary. This is intended for development only and its shape is not part of the stable template contract. (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` @@ -1075,7 +1075,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``request`` - Full information about the request + Dictionary with request details: ``url``, ``path``, ``full_path``, ``host`` and ``args`` where ``args`` maps query string parameter names to their values. ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request`` @@ -1097,7 +1097,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``query`` - Details of the underlying SQL query + Details of the underlying SQL query as a dictionary with ``sql`` and ``params`` keys. ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query`` @@ -1120,7 +1120,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``metadata`` - Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + Metadata dictionary for the table, database or stored query. Table and row metadata include a ``columns`` dictionary mapping column names to descriptions; stored query metadata returns the stored query configuration. (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` @@ -1151,7 +1151,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``extras`` - List of ?_extra= blocks that can be used on this page + List of ``?_extra=`` blocks that can be used on this page. Each item has ``name``, ``description``, ``toggle_url`` and ``selected`` keys. Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. diff --git a/docs/template_context.rst b/docs/template_context.rst new file mode 100644 index 00000000..52ccca49 --- /dev/null +++ b/docs/template_context.rst @@ -0,0 +1,506 @@ +.. _template_context: + +Template context +================ + +This page documents the variables that are available to custom templates +for each of Datasette's core pages. See :ref:`customization_custom_templates` +for how to provide your own templates. + +The variables documented here are a stable contract: custom templates that +use them will continue to work across Datasette releases, up until the next +major version (Datasette 2.0). Anything present in the template context but +not documented on this page is not part of that contract and may change or +be removed in any release. + +You can inspect the full context for any page by starting Datasette with +``--setting template_debug 1`` and adding ``?_context=1`` to the page URL. + +.. [[[cog + from template_context_doc import template_context + template_context(cog) +.. ]]] + +Base context +------------ + +These variables are available on every page rendered by Datasette, including pages rendered by plugins that use :ref:`datasette.render_template() `. Plugins can add additional variables using the :ref:`plugin_hook_extra_template_vars` hook. + +``request`` + The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``. + +``crumb_items`` + Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries. + +``urls`` + Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`. + +``actor`` + The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins. + +``menu_links`` + Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`. + +``display_actor`` + Function that accepts an actor dictionary and returns the display string used in the navigation menu. + +``show_logout`` + True if the logout link should be shown in the navigation menu + +``app_css_hash`` + Hash of Datasette's app.css contents, used for cache busting + +``edit_tools_js_hash`` + Hash of Datasette's edit-tools.js contents, used for cache busting + +``table_js_hash`` + Hash of Datasette's table.js contents, used for cache busting + +``zip`` + Python's ``zip()`` builtin, made available to template logic + +``body_scripts`` + List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in ``