diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 5fb6b473..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -19,23 +19,38 @@ import weakref import pytest -from datasette.app import Datasette - _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( "datasette_active_instances", default=None ) -_original_init = Datasette.__init__ +_original_init = None -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init -Datasette.__init__ = _tracking_init +def pytest_configure(config): + if _enabled(config): + _install_tracking() def pytest_addoption(parser): diff --git a/datasette/extras.py b/datasette/extras.py new file mode 100644 index 00000000..5cab52a4 --- /dev/null +++ b/datasette/extras.py @@ -0,0 +1,118 @@ +import re +from dataclasses import dataclass +from enum import Enum +from typing import ClassVar + +from asyncinject import Registry + + +def extra_names_from_request(request): + extra_bits = request.args.getlist("_extra") + extras = set() + for bit in extra_bits: + extras.update(part for part in bit.split(",") if part) + return extras + + +class ExtraScope(Enum): + TABLE = "table" + ROW = "row" + QUERY = "query" + + +@dataclass(frozen=True) +class ExtraExample: + path: str | None = None + key: str | None = None + value: object | None = None + note: str | None = None + + +class Provider: + name: ClassVar[str | None] = None + scopes: ClassVar[set[ExtraScope]] = set() + public: ClassVar[bool] = False + + @classmethod + def key(cls): + return cls.name or _camel_to_snake(cls.__name__) + + @classmethod + def available_for(cls, scope): + return scope in cls.scopes + + async def resolve(self, context): + raise NotImplementedError + + +class Extra(Provider): + description: ClassVar[str | None] = None + example: ClassVar[ExtraExample | None] = None + examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} + public: ClassVar[bool] = True + expensive: ClassVar[bool] = False + docs_note: ClassVar[str | None] = None + + @classmethod + def example_for_scope(cls, scope): + return cls.examples.get(scope, cls.example) + + +class ExtraRegistry: + def __init__(self, classes): + self.classes = list(classes) + self.classes_by_name = {cls.key(): cls for cls in self.classes} + # Lazily-built shared state, keyed by scope. Safe to share across + # requests because Extra instances are stateless and asyncinject's + # Registry keeps per-call state local to each resolve_multi() call. + # If extras classes ever become registerable at runtime (e.g. via a + # plugin hook) these caches will need invalidating. + self._scope_registries = {} + self._allowed_names = {} + + def classes_for_scope(self, scope, include_internal=True): + classes = [ + cls + for cls in self.classes + if cls.available_for(scope) and (include_internal or cls.public) + ] + return classes + + def public_classes_for_scope(self, scope): + return self.classes_for_scope(scope, include_internal=False) + + def _registry_for_scope(self, scope): + registry = self._scope_registries.get(scope) + if registry is None: + registry = Registry() + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + self._scope_registries[scope] = registry + return registry + + def _allowed_names_for_scope(self, scope, include_internal): + key = (scope, include_internal) + names = self._allowed_names.get(key) + if names is None: + names = { + cls.key() + for cls in self.classes_for_scope( + scope, include_internal=include_internal + ) + } + self._allowed_names[key] = names + return names + + async def resolve(self, requested, context, scope, include_internal=False): + allowed_names = self._allowed_names_for_scope(scope, include_internal) + requested_names = [name for name in requested if name in allowed_names] + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names, results={"context": context} + ) + return {name: resolved[name] for name in requested_names} + + +def _camel_to_snake(name): + name = re.sub(r"(Extra|Provider)$", "", name) + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..f40e3dbb 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,4 +1,5 @@ import json +from datasette.extras import extra_names_from_request from datasette.utils import ( value_as_boolean, remove_infinites, @@ -108,7 +109,7 @@ def json_renderer(request, args, data, error, truncated=None): # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 - if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + if isinstance(data, dict) and "columns" not in extra_names_from_request(request): data.pop("columns", None) # Handle _nl option for _shape=array diff --git a/datasette/views/database.py b/datasette/views/database.py index 66887f9b..f1756863 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,6 +11,7 @@ import sqlite_utils import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict @@ -38,6 +39,11 @@ from datasette.plugins import pm from .base import BaseView, DatasetteError, View, _error, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .table_extras import ( + QueryExtraContext, + resolve_query_extras, + table_extra_registry, +) from . import Context @@ -606,11 +612,13 @@ class QueryView(View): ) else: - await datasette.ensure_permission( + visible, private = await datasette.check_visibility( + request.actor, action="execute-sql", resource=DatabaseResource(database=database), - actor=request.actor, ) + if not visible: + raise Forbidden("execute-sql") # Flattened because of ?sql=&name1=value1&name2=value2 feature params = {key: request.args.get(key) for key in request.args} @@ -692,6 +700,13 @@ class QueryView(View): except DatasetteError: raise + async def query_metadata(): + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) + return metadata + return await datasette.get_database_metadata(database) + # Handle formats from plugins if format_ == "csv": if not sql: @@ -704,6 +719,25 @@ class QueryView(View): return await stream_csv(datasette, fetch_data_for_csv, request, db.name) elif format_ in datasette.renderers.keys(): + data = {"ok": True, "rows": rows, "columns": columns} + extras = extra_names_from_request(request) + if extras: + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=named_parameter_values, + query_name=stored_query.name if stored_query else None, + metadata=await query_metadata(), + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -721,7 +755,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"ok": True, "rows": rows, "columns": columns}, + data=data, ) if asyncio.iscoroutine(result): result = await result @@ -778,11 +812,7 @@ class QueryView(View): ) } ) - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) - + metadata = await query_metadata() renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( diff --git a/datasette/views/row.py b/datasette/views/row.py index 4eacfe49..c300758b 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -14,7 +14,9 @@ from datasette.plugins import pm import json import markupsafe import sqlite_utils -from .table import display_columns_and_rows, _get_extras +from datasette.extras import extra_names_from_request +from .table import display_columns_and_rows +from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry class RowView(DataView): @@ -164,60 +166,27 @@ class RowView(DataView): "primary_key_values": pk_values, } - # Handle _extra parameter (new style) - extras = _get_extras(request) - - # Also support legacy _extras parameter for backward compatibility - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - extras.add("foreign_key_tables") + extras = extra_names_from_request(request) # Process extras - if "foreign_key_tables" in extras: - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - if "render_cell" in extras: - # Call render_cell plugin hook for each cell - ct_map = await self.ds.get_column_types(database, table) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, columns): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table, - pks=resolved.pks, - database=database, - datasette=self.ds, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - data["render_cell"] = rendered_rows + row_extra_context = RowExtraContext( + datasette=self.ds, + request=request, + db=db, + database_name=database, + table_name=table, + private=private, + rows=rows, + columns=columns, + pks=pks, + pk_values=pk_values, + sql=resolved.sql, + params=resolved.params, + extras=extras, + extra_registry=table_extra_registry, + foreign_key_tables=self.foreign_key_tables, + ) + data.update(await resolve_row_extras(extras, row_extra_context)) return ( data, diff --git a/datasette/views/table.py b/datasette/views/table.py index 4df1e1b4..65388c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,11 +3,10 @@ import itertools import json import urllib -from asyncinject import Registry import markupsafe +from datasette.extras import extra_names_from_request from datasette.plugins import pm -from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, @@ -46,6 +45,12 @@ from datasette.filters import Filters import sqlite_utils from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView +from .table_extras import ( + TABLE_EXTRA_BUNDLES, + TableExtraContext, + resolve_table_extras, + table_extra_registry, +) LINK_WITH_LABEL = ( '{label} {id}' @@ -849,14 +854,6 @@ class TableDropView(BaseView): return Response.json({"ok": True}, status=200) -def _get_extras(request): - extra_bits = request.args.getlist("_extra") - extras = set() - for bit in extra_bits: - extras.update(bit.split(",")) - return extras - - async def _columns_to_select(table_columns, pks, request): columns = list(table_columns) if "_col" in request.args: @@ -1460,7 +1457,7 @@ async def table_view_data( rows = rows[:page_size] # Resolve extras - extras = _get_extras(request) + extras = extra_names_from_request(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): extras.add("facet_results") if request.args.get("_shape") == "object": @@ -1468,559 +1465,65 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) - async def extra_count_sql(): - return count_sql - - async def extra_count(): - "Total count of rows matching these filters" - # Calculate the total count for this query - count = None - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - # We can use a previously cached table row count - try: - count = datasette.inspect_data[database_name]["tables"][table_name][ - "count" - ] - except KeyError: - pass - - # Otherwise run a select count(*) ... - if count_sql and count is None and not nocount: - count_sql_limited = ( - f"select count(*) from (select * {from_sql} limit 10001)" - ) - try: - count_rows = list(await db.execute(count_sql_limited, from_sql_params)) - count = count_rows[0][0] - except QueryInterrupted: - pass - return count - - async def facet_instances(extra_count): - facet_instances = [] - facet_classes = list( - itertools.chain.from_iterable(pm.hook.register_facet_classes()) - ) - for facet_class in facet_classes: - facet_instances.append( - facet_class( - datasette, - request, - database_name, - sql=sql_no_order_no_limit, - params=params, - table=table_name, - table_config=table_metadata, - row_count=extra_count, - ) - ) - return facet_instances - - async def extra_facet_results(facet_instances): - "Results of facets calculated against this data" - facet_results = {} - facets_timed_out = [] - - if not nofacet: - # Run them in parallel - facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await run_sequential(*facet_awaitables) - for ( - instance_facet_results, - instance_facets_timed_out, - ) in facet_awaitable_results: - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) - - return { - "results": facet_results, - "timed_out": facets_timed_out, - } - - async def extra_suggested_facets(facet_instances): - "Suggestions for facets that might return interesting results" - suggested_facets = [] - # Calculate suggested facets - if ( - datasette.setting("suggest_facets") - and datasette.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - # Run them in parallel - facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await run_sequential(*facet_suggest_awaitables): - suggested_facets.extend(suggest_result) - return suggested_facets - # Faceting if not datasette.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): raise BadRequest("_facet= is not allowed") - # human_description_en combines filters AND search, if provided - async def extra_human_description_en(): - "Human-readable description of the filters" - human_description_en = filters.human_description_en( - extra=extra_human_descriptions - ) - if sort or sort_desc: - human_description_en = " ".join( - [b for b in [human_description_en, sorted_by] if b] - ) - return human_description_en - - if sort or sort_desc: - sorted_by = "sorted by {}{}".format( - (sort or sort_desc), " descending" if sort_desc else "" - ) - - async def extra_next_url(): - "Full URL for the next page of results" - return next_url - - async def extra_columns(): - "Column names returned by this query" - return columns - - async def extra_all_columns(): - "All columns in the table, regardless of _col/_nocol filtering" - return list(table_columns) - - async def extra_primary_keys(): - "Primary keys for this table" - return pks - - async def extra_actions(): - async def actions(): - links = [] - kwargs = { - "datasette": datasette, - "database": database_name, - "actor": request.actor, - "request": request, - } - if is_view: - kwargs["view"] = table_name - method = pm.hook.view_actions - else: - kwargs["table"] = table_name - method = pm.hook.table_actions - for hook in method(**kwargs): - extra_links = await await_me_maybe(hook) - if extra_links: - links.extend(extra_links) - return links - - return actions - - async def extra_is_view(): - return is_view - - async def extra_debug(): - "Extra debug information" - return { - "resolved": repr(resolved), - "url_vars": request.url_vars, - "nofacet": nofacet, - "nosuggest": nosuggest, - } - - async def extra_request(): - "Full information about the request" - return { - "url": request.url, - "path": request.path, - "full_path": request.full_path, - "host": request.host, - "args": request.args._data, - } - - async def run_display_columns_and_rows(): - display_columns, display_rows = await display_columns_and_rows( - datasette, - database_name, - table_name, - results.description, - rows, - link_column=not is_view, - truncate_cells=datasette.setting("truncate_cells_html"), - sortable_columns=sortable_columns, - request=request, - ) - return { - "columns": display_columns, - "rows": display_rows, - } - - async def extra_display_columns(run_display_columns_and_rows): - return run_display_columns_and_rows["columns"] - - async def extra_display_rows(run_display_columns_and_rows): - return run_display_columns_and_rows["rows"] - - async def extra_render_cell(): - "Rendered HTML for each cell using the render_cell plugin hook" - pks_for_display = pks if pks else (["rowid"] if not is_view else []) - col_names = [col[0] for col in results.description] - ct_map = await datasette.get_column_types(database_name, table_name) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, col_names): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table_name, - database=database_name, - datasette=datasette, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table_name, - pks=pks_for_display, - database=database_name, - datasette=datasette, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - return rendered_rows - - async def extra_query(): - "Details of the underlying SQL query" - return { - "sql": sql, - "params": params, - } - - async def extra_column_types(): - "Column type assignments for this table" - ct_map = await datasette.get_column_types(database_name, table_name) - return { - col_name: { - "type": ct.name, - "config": ct.config, - } - for col_name, ct in ct_map.items() - } - - async def extra_set_column_type_ui(): - "Column type UI metadata for this table" - if is_view: - return None - - if not await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return None - - column_details = await datasette._get_resource_column_details( - database_name, table_name - ) - ct_map = await datasette.get_column_types(database_name, table_name) - columns = {} - for column_name, column_detail in column_details.items(): - current = ct_map.get(column_name) - columns[column_name] = { - "current": ( - {"type": current.name, "config": current.config} - if current is not None - else None - ), - "options": [ - { - "name": name, - "description": ct_cls.description, - } - for name, ct_cls in sorted(datasette._column_types.items()) - if datasette._column_type_is_applicable(ct_cls, column_detail) - ], - } - return { - "path": "{}/-/set-column-type".format( - datasette.urls.table(database_name, table_name) - ), - "columns": columns, - } - - async def extra_metadata(): - "Metadata about the table and database" - tablemetadata = await datasette.get_resource_metadata(database_name, table_name) - - rows = await datasette.get_internal_database().execute( - """ - SELECT - column_name, - value - FROM metadata_columns - WHERE database_name = ? - AND resource_name = ? - AND key = 'description' - """, - [database_name, table_name], - ) - tablemetadata["columns"] = dict(rows) - return tablemetadata - - async def extra_database(): - return database_name - - async def extra_table(): - return table_name - - async def extra_database_color(): - return db.color - - async def extra_form_hidden_args(): - form_hidden_args = [] - for key in request.args: - if ( - key.startswith("_") - and key not in ("_sort", "_sort_desc", "_search", "_next") - and "__" not in key - ): - for value in request.args.getlist(key): - form_hidden_args.append((key, value)) - return form_hidden_args - - async def extra_filters(): - return filters - - async def extra_custom_table_templates(): - return [ - f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - "_table.html", - ] - - async def extra_sorted_facet_results(extra_facet_results): - facet_configs = table_metadata.get("facets", []) - if facet_configs: - # Build ordered list of facet names from metadata config - metadata_facet_names = [] - for fc in facet_configs: - if isinstance(fc, str): - metadata_facet_names.append(fc) - elif isinstance(fc, dict): - metadata_facet_names.append(list(fc.values())[0]) - metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} - metadata_facets = [] - request_facets = [] - for f in extra_facet_results["results"].values(): - if f["name"] in metadata_order: - metadata_facets.append(f) - else: - request_facets.append(f) - metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) - request_facets.sort( - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - return metadata_facets + request_facets - else: - return sorted( - extra_facet_results["results"].values(), - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - - async def extra_table_definition(): - return await db.get_table_definition(table_name) - - async def extra_view_definition(): - return await db.get_view_definition(table_name) - - async def extra_renderers(extra_expandable_columns, extra_query): - renderers = {} - url_labels_extra = {} - if extra_expandable_columns: - url_labels_extra = {"_labels": "on"} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=columns or [], - rows=rows or [], - sql=extra_query.get("sql", None), - query_name=None, - database=database_name, - table=table_name, - request=request, - view_name="table", - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - return renderers - - async def extra_private(): - return private - - async def extra_expandable_columns(): - expandables = [] - db = datasette.databases[database_name] - for fk in await db.foreign_keys_for_table(table_name): - label_column = await db.label_column_for_table(fk["other_table"]) - expandables.append((fk, label_column)) - return expandables - - async def extra_extras(): - "Available ?_extra= blocks" - all_extras = [ - (key[len("extra_") :], fn.__doc__) - for key, fn in registry._registry.items() - if key.startswith("extra_") - ] - return [ - { - "name": name, - "description": doc, - "toggle_url": datasette.absolute_url( - request, - datasette.urls.path( - path_with_added_args(request, {"_extra": name}) - if name not in extras - else path_with_removed_args(request, {"_extra": name}) - ), - ), - "selected": name in extras, - } - for name, doc in all_extras - ] - - async def extra_facets_timed_out(extra_facet_results): - return extra_facet_results["timed_out"] - - bundles = { - "html": [ - "suggested_facets", - "facet_results", - "facets_timed_out", - "count", - "count_sql", - "human_description_en", - "next_url", - "metadata", - "query", - "columns", - "display_columns", - "display_rows", - "database", - "table", - "database_color", - "actions", - "filters", - "renderers", - "custom_table_templates", - "sorted_facet_results", - "table_definition", - "view_definition", - "is_view", - "private", - "primary_keys", - "all_columns", - "expandable_columns", - "form_hidden_args", - "set_column_type_ui", - ] - } - - for key, values in bundles.items(): + for key, values in TABLE_EXTRA_BUNDLES.items(): if f"_{key}" in extras: extras.update(values) extras.discard(f"_{key}") - registry = Registry( - extra_count, - extra_count_sql, - extra_facet_results, - extra_facets_timed_out, - extra_suggested_facets, - facet_instances, - extra_human_description_en, - extra_next_url, - extra_columns, - extra_all_columns, - extra_primary_keys, - run_display_columns_and_rows, - extra_display_columns, - extra_display_rows, - extra_render_cell, - extra_debug, - extra_request, - extra_query, - extra_column_types, - extra_set_column_type_ui, - extra_metadata, - extra_extras, - extra_database, - extra_table, - extra_database_color, - extra_actions, - extra_filters, - extra_renderers, - extra_custom_table_templates, - extra_sorted_facet_results, - extra_table_definition, - extra_view_definition, - extra_is_view, - extra_private, - extra_expandable_columns, - extra_form_hidden_args, + table_extra_context = TableExtraContext( + datasette=datasette, + request=request, + resolved=resolved, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + table_columns=table_columns, + pks=pks, + count_sql=count_sql, + from_sql=from_sql, + from_sql_params=from_sql_params, + nocount=nocount, + nofacet=nofacet, + nosuggest=nosuggest, + next_arg=request.args.get("_next"), + next_url=next_url, + sql=sql, + sql_no_order_no_limit=sql_no_order_no_limit, + params=params, + table_metadata=table_metadata, + filters=filters, + extra_human_descriptions=extra_human_descriptions, + sort=sort, + sort_desc=sort_desc, + sortable_columns=sortable_columns, + extras=extras, + extra_registry=table_extra_registry, + display_columns_and_rows=display_columns_and_rows, + run_sequential=run_sequential, ) - results = await registry.resolve_multi( - ["extra_{}".format(extra) for extra in extras] - ) data = { "ok": True, "next": next_value and str(next_value) or None, } data.update( - { - key.replace("extra_", ""): value - for key, value in results.items() - if key.startswith("extra_") and key.replace("extra_", "") in extras - } + await resolve_table_extras( + extras, + table_extra_context, + # The HTML view needs extras that are not JSON serializable + include_internal=bool(extra_extras), + ) ) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py new file mode 100644 index 00000000..ce1d7bdf --- /dev/null +++ b/datasette/views/table_extras.py @@ -0,0 +1,1049 @@ +import itertools +from dataclasses import dataclass + +from datasette.database import QueryInterrupted +from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider +from datasette.plugins import pm +from datasette.resources import TableResource +from datasette.utils import ( + await_me_maybe, + call_with_supported_arguments, + path_with_added_args, + path_with_format, + path_with_removed_args, + to_css_class, +) + + +@dataclass(frozen=True) +class TableExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + is_view: bool + private: bool + rows: list + columns: list + results_description: list + table_columns: list + pks: list + count_sql: str + from_sql: str + from_sql_params: dict + nocount: object + nofacet: object + nosuggest: object + next_arg: object + next_url: str | None + sql: str + sql_no_order_no_limit: str + params: dict + table_metadata: dict + filters: object + extra_human_descriptions: list + sort: str | None + sort_desc: str | None + sortable_columns: set + extras: set + extra_registry: ExtraRegistry + display_columns_and_rows: object + run_sequential: object + query_name: str | None = None + scope: ExtraScope = ExtraScope.TABLE + + +@dataclass(frozen=True) +class RowExtraContext: + datasette: object + request: object + db: object + database_name: str + table_name: str + private: bool + rows: list + columns: list + pks: list + pk_values: list + sql: str + params: dict + extras: set + extra_registry: ExtraRegistry + foreign_key_tables: object + is_view: bool = False + scope: ExtraScope = ExtraScope.ROW + + +@dataclass(frozen=True) +class QueryExtraContext: + datasette: object + request: object + db: object + database_name: str + private: bool + rows: list + columns: list + sql: str | None + params: dict + query_name: str | None + metadata: dict + extras: set + extra_registry: ExtraRegistry + table_name: str | None = None + is_view: bool = False + pks: list | None = None + scope: ExtraScope = ExtraScope.QUERY + + +class CountSqlExtra(Extra): + description = "SQL query used to calculate the total count" + example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return context.count_sql + + +class CountExtra(Extra): + description = "Total count of rows matching these filters" + example = ExtraExample("/fixtures/facetable.json?_extra=count") + scopes = {ExtraScope.TABLE} + expensive = True + + async def resolve(self, context): + count = None + if ( + not context.db.is_mutable + and context.datasette.inspect_data + and context.count_sql == f"select count(*) from {context.table_name} " + ): + try: + count = context.datasette.inspect_data[context.database_name]["tables"][ + context.table_name + ]["count"] + except KeyError: + pass + + if context.count_sql and count is None and not context.nocount: + count_sql_limited = ( + f"select count(*) from (select * {context.from_sql} limit 10001)" + ) + try: + count_rows = list( + await context.db.execute(count_sql_limited, context.from_sql_params) + ) + count = count_rows[0][0] + except QueryInterrupted: + pass + return count + + +class FacetInstancesProvider(Provider): + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, count): + facet_instances = [] + facet_classes = list( + itertools.chain.from_iterable(pm.hook.register_facet_classes()) + ) + for facet_class in facet_classes: + facet_instances.append( + facet_class( + context.datasette, + context.request, + context.database_name, + sql=context.sql_no_order_no_limit, + params=context.params, + table=context.table_name, + table_config=context.table_metadata, + row_count=count, + ) + ) + return facet_instances + + +class FacetResultsExtra(Extra): + description = "Results of facets calculated against this data" + example = ExtraExample( + value={ + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + {"value": "CA", "label": "CA", "count": 10}, + {"value": "MI", "label": "MI", "count": 4}, + ], + } + }, + "timed_out": [], + }, + note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", + ) + scopes = {ExtraScope.TABLE} + expensive = True + + async def resolve(self, context, facet_instances): + facet_results = {} + facets_timed_out = [] + + if not context.nofacet: + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await context.run_sequential(*facet_awaitables) + for ( + instance_facet_results, + instance_facets_timed_out, + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) + + return { + "results": facet_results, + "timed_out": facets_timed_out, + } + + +class FacetsTimedOutExtra(Extra): + description = "Facet calculations that timed out" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + ) + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, facet_results): + return facet_results["timed_out"] + + +class SuggestedFacetsExtra(Extra): + description = "Suggestions for facets that might return interesting results" + example = ExtraExample( + value=[ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state", + } + ], + note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", + ) + scopes = {ExtraScope.TABLE} + expensive = True + + async def resolve(self, context, facet_instances): + suggested_facets = [] + if ( + context.datasette.setting("suggest_facets") + and context.datasette.setting("allow_facet") + and not context.next_arg + and not context.nofacet + and not context.nosuggest + ): + facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] + for suggest_result in await context.run_sequential( + *facet_suggest_awaitables + ): + suggested_facets.extend(suggest_result) + return suggested_facets + + +class HumanDescriptionEnExtra(Extra): + description = "Human-readable description of the filters" + example = ExtraExample( + "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" + ) + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + human_description_en = context.filters.human_description_en( + extra=context.extra_human_descriptions + ) + if context.sort or context.sort_desc: + sorted_by = "sorted by {}{}".format( + (context.sort or context.sort_desc), + " descending" if context.sort_desc else "", + ) + human_description_en = " ".join( + [b for b in [human_description_en, sorted_by] if b] + ) + return human_description_en + + +class NextUrlExtra(Extra): + description = "Full URL for the next page of results" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return context.next_url + + +class ColumnsExtra(Extra): + description = "Column names returned by this query" + example = ExtraExample("/fixtures/facetable.json?_extra=columns") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=columns" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + return context.columns + + +class AllColumnsExtra(Extra): + description = "All columns in the table, regardless of _col/_nocol filtering" + example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return list(context.table_columns) + + +class PrimaryKeysExtra(Extra): + description = "Primary keys for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=primary_keys" + ) + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW} + + async def resolve(self, context): + return context.pks + + +class ActionsExtra(Extra): + description = "Table or view actions made available by plugin hooks" + scopes = {ExtraScope.TABLE} + # Returns an async function for the HTML templates - not JSON serializable + public = False + + async def resolve(self, context): + async def actions(): + links = [] + kwargs = { + "datasette": context.datasette, + "database": context.database_name, + "actor": context.request.actor, + "request": context.request, + } + if context.is_view: + kwargs["view"] = context.table_name + method = pm.hook.view_actions + else: + kwargs["table"] = context.table_name + method = pm.hook.table_actions + for hook in method(**kwargs): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + return actions + + +class IsViewExtra(Extra): + description = "Whether this resource is a view instead of a table" + example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return context.is_view + + +class DebugExtra(Extra): + description = "Extra debug information" + example = ExtraExample("/fixtures/facetable.json?_extra=debug") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=debug" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + debug = { + "url_vars": context.request.url_vars, + } + if context.scope == ExtraScope.TABLE: + debug["resolved"] = repr(context.resolved) + debug["nofacet"] = context.nofacet + debug["nosuggest"] = context.nosuggest + elif context.scope == ExtraScope.ROW: + debug["resolved"] = { + "table": context.table_name, + "sql": context.sql, + "params": context.params, + "pks": context.pks, + "pk_values": context.pk_values, + } + return debug + + +class RequestExtra(Extra): + description = "Full information about the request" + example = ExtraExample("/fixtures/facetable.json?_extra=request") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=request" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + return { + "url": context.request.url, + "path": context.request.path, + "full_path": context.request.full_path, + "host": context.request.host, + "args": context.request.args._data, + } + + +class DisplayColumnsAndRowsProvider(Provider): + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + display_columns, display_rows = await context.display_columns_and_rows( + context.datasette, + context.database_name, + context.table_name, + context.results_description, + context.rows, + link_column=not context.is_view, + truncate_cells=context.datasette.setting("truncate_cells_html"), + sortable_columns=context.sortable_columns, + request=context.request, + ) + return { + "columns": display_columns, + "rows": display_rows, + } + + +class DisplayColumnsExtra(Extra): + description = "Column metadata used by the HTML table display" + example = ExtraExample( + value=[ + { + "name": "pk", + "sortable": True, + "is_pk": True, + "type": "INTEGER", + "notnull": 0, + }, + { + "name": "created", + "sortable": True, + "is_pk": False, + "type": "TEXT", + "notnull": 0, + "description": None, + "column_type": None, + "column_type_config": None, + }, + ], + note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", + ) + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["columns"] + + +class DisplayRowsExtra(Extra): + description = "Row data formatted for the HTML table display" + scopes = {ExtraScope.TABLE} + # Contains markupsafe/sqlite3.Row values - not JSON serializable + public = False + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["rows"] + + +class RenderCellExtra(Extra): + description = "Rendered HTML for each cell using the render_cell plugin hook" + example = ExtraExample( + value={ + "rows": [ + {"id": 1, "content": "hello"}, + {"id": 4, "content": "RENDER_CELL_DEMO"}, + ], + "render_cell": [ + {}, + {"content": "Custom rendered HTML"}, + ], + }, + note=( + "The ``render_cell`` array has one item per row, in the same order as " + "the ``rows`` array. Each object is keyed by column name. Only columns " + "whose rendered value differs from the default are included." + ), + ) + examples = { + ExtraScope.ROW: ExtraExample( + value={ + "rows": [{"id": 4, "content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item for the requested row. " + "The object is keyed by column name. Only columns whose rendered " + "value differs from the default are included." + ), + ), + ExtraScope.QUERY: ExtraExample( + value={ + "rows": [{"content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item per query result row, in " + "the same order as the ``rows`` array. Each object is keyed by " + "column name. Only columns whose rendered value differs from the " + "default are included." + ), + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + table_name = context.table_name + pks_for_display = context.pks or ( + ["rowid"] if table_name and not context.is_view else [] + ) + ct_map = ( + await context.datasette.get_column_types(context.database_name, table_name) + if table_name + else {} + ) + rendered_rows = [] + for row in context.rows: + rendered_row = {} + for value, column in zip(row, context.columns): + ct = ct_map.get(column) + plugin_display_value = None + if ct: + candidate = await ct.render_cell( + value=value, + column=column, + table=table_name, + database=context.database_name, + datasette=context.datasette, + request=context.request, + ) + if candidate is not None: + plugin_display_value = candidate + if plugin_display_value is None: + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=table_name, + pks=pks_for_display, + database=context.database_name, + datasette=context.datasette, + request=context.request, + column_type=ct, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + rendered_row[column] = str(plugin_display_value) + rendered_rows.append(rendered_row) + return rendered_rows + + +class QueryExtra(Extra): + description = "Details of the underlying SQL query" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=query" + ), + ExtraScope.QUERY: [ + ExtraExample("/fixtures/-/query.json?sql=select+1+as+one&_extra=query"), + ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), + ], + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + return { + "sql": context.sql, + "params": context.params, + } + + +class ColumnTypesExtra(Extra): + description = "Column type assignments for this table" + example = ExtraExample(value={}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} + + async def resolve(self, context): + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + return { + col_name: { + "type": ct.name, + "config": ct.config, + } + for col_name, ct in ct_map.items() + } + + +class SetColumnTypeUiExtra(Extra): + description = "Column type UI metadata for this table" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + if context.is_view: + return None + + if not await context.datasette.allowed( + action="set-column-type", + resource=TableResource( + database=context.database_name, table=context.table_name + ), + actor=context.request.actor, + ): + return None + + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + columns = {} + for column_name, column_detail in column_details.items(): + current = ct_map.get(column_name) + columns[column_name] = { + "current": ( + {"type": current.name, "config": current.config} + if current is not None + else None + ), + "options": [ + { + "name": name, + "description": ct_cls.description, + } + for name, ct_cls in sorted(context.datasette._column_types.items()) + if context.datasette._column_type_is_applicable( + ct_cls, column_detail + ) + ], + } + return { + "path": "{}/-/set-column-type".format( + context.datasette.urls.table(context.database_name, context.table_name) + ), + "columns": columns, + } + + +class MetadataExtra(Extra): + description = "Metadata about the table, database or stored query" + example = ExtraExample("/fixtures/facetable.json?_extra=metadata") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=metadata" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + if context.scope == ExtraScope.QUERY: + return context.metadata + + tablemetadata = await context.datasette.get_resource_metadata( + context.database_name, context.table_name + ) + + rows = await context.datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM metadata_columns + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [context.database_name, context.table_name], + ) + tablemetadata["columns"] = dict(rows) + return tablemetadata + + +class DatabaseExtra(Extra): + description = "Database name" + example = ExtraExample("/fixtures/facetable.json?_extra=database") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + return context.database_name + + +class TableExtra(Extra): + description = "Table name" + example = ExtraExample("/fixtures/facetable.json?_extra=table") + examples = { + ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW} + + async def resolve(self, context): + return context.table_name + + +class DatabaseColorExtra(Extra): + description = "Color assigned to the database" + example = ExtraExample("/fixtures/facetable.json?_extra=database_color") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database_color" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + return context.db.color + + +class FormHiddenArgsExtra(Extra): + description = "Hidden form arguments used by the HTML table interface" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" + ) + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + form_hidden_args = [] + for key in context.request.args: + if ( + key.startswith("_") + and key not in ("_sort", "_sort_desc", "_search", "_next") + and "__" not in key + ): + for value in context.request.args.getlist(key): + form_hidden_args.append((key, value)) + return form_hidden_args + + +class FiltersExtra(Extra): + description = "Filters object used by the HTML table interface" + scopes = {ExtraScope.TABLE} + # Returns a Filters instance for the HTML templates - not JSON serializable + public = False + + async def resolve(self, context): + return context.filters + + +class CustomTableTemplatesExtra(Extra): + description = "Custom template names considered for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return [ + f"_table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + f"_table-table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + "_table.html", + ] + + +class SortedFacetResultsExtra(Extra): + description = "Facet results sorted for display" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" + ) + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, facet_results): + facet_configs = context.table_metadata.get("facets", []) + if facet_configs: + metadata_facet_names = [] + for fc in facet_configs: + if isinstance(fc, str): + metadata_facet_names.append(fc) + elif isinstance(fc, dict): + metadata_facet_names.append(list(fc.values())[0]) + metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} + metadata_facets = [] + request_facets = [] + for f in facet_results["results"].values(): + if f["name"] in metadata_order: + metadata_facets.append(f) + else: + request_facets.append(f) + metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) + request_facets.sort( + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + return metadata_facets + request_facets + else: + return sorted( + facet_results["results"].values(), + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + + +class TableDefinitionExtra(Extra): + description = "SQL definition for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return await context.db.get_table_definition(context.table_name) + + +class ViewDefinitionExtra(Extra): + description = "SQL definition for this view" + example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + return await context.db.get_view_definition(context.table_name) + + +class RenderersExtra(Extra): + description = "Alternative output renderers available for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=renderers") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, expandable_columns, query): + renderers = {} + url_labels_extra = {} + if expandable_columns: + url_labels_extra = {"_labels": "on"} + table_name = context.table_name + view_name = "table" if context.scope == ExtraScope.TABLE else "database" + for key, (_, can_render) in context.datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=context.datasette, + columns=context.columns or [], + rows=context.rows or [], + sql=query.get("sql", None), + query_name=context.query_name, + database=context.database_name, + table=table_name, + request=context.request, + view_name=view_name, + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = context.datasette.urls.path( + path_with_format( + request=context.request, + path=context.request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + return renderers + + +class PrivateExtra(Extra): + description = "Whether this resource is private to the current actor" + example = ExtraExample("/fixtures/facetable.json?_extra=private") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=private" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ), + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + return context.private + + +class ExpandableColumnsExtra(Extra): + description = "Foreign key columns that can be expanded with labels" + example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + expandables = [] + db = context.datasette.databases[context.database_name] + for fk in await db.foreign_keys_for_table(context.table_name): + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) + return expandables + + +class ForeignKeyTablesExtra(Extra): + description = "Tables that link to this row using foreign keys" + example = ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + ) + scopes = {ExtraScope.ROW} + + async def resolve(self, context): + return await context.foreign_key_tables( + context.database_name, context.table_name, context.pk_values + ) + + +class ExtrasExtra(Extra): + description = "Available ?_extra= blocks" + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} + + async def resolve(self, context): + all_extras = [ + (cls.key(), cls.description) + for cls in context.extra_registry.public_classes_for_scope(context.scope) + ] + return [ + { + "name": name, + "description": description, + "toggle_url": context.datasette.absolute_url( + context.request, + context.datasette.urls.path( + path_with_added_args(context.request, {"_extra": name}) + if name not in context.extras + else path_with_removed_args(context.request, {"_extra": name}) + ), + ), + "selected": name in context.extras, + } + for name, description in all_extras + ] + + +TABLE_EXTRA_BUNDLES = { + "html": [ + "suggested_facets", + "facet_results", + "facets_timed_out", + "count", + "count_sql", + "human_description_en", + "next_url", + "metadata", + "query", + "columns", + "display_columns", + "display_rows", + "database", + "table", + "database_color", + "actions", + "filters", + "renderers", + "custom_table_templates", + "sorted_facet_results", + "table_definition", + "view_definition", + "is_view", + "private", + "primary_keys", + "all_columns", + "expandable_columns", + "form_hidden_args", + "set_column_type_ui", + ] +} + + +TABLE_EXTRA_CLASSES = [ + CountExtra, + CountSqlExtra, + FacetResultsExtra, + FacetsTimedOutExtra, + SuggestedFacetsExtra, + FacetInstancesProvider, + HumanDescriptionEnExtra, + NextUrlExtra, + ColumnsExtra, + AllColumnsExtra, + PrimaryKeysExtra, + DisplayColumnsAndRowsProvider, + DisplayColumnsExtra, + DisplayRowsExtra, + RenderCellExtra, + DebugExtra, + RequestExtra, + QueryExtra, + ColumnTypesExtra, + SetColumnTypeUiExtra, + MetadataExtra, + ExtrasExtra, + DatabaseExtra, + TableExtra, + DatabaseColorExtra, + ActionsExtra, + FiltersExtra, + RenderersExtra, + CustomTableTemplatesExtra, + SortedFacetResultsExtra, + TableDefinitionExtra, + ViewDefinitionExtra, + IsViewExtra, + PrivateExtra, + ExpandableColumnsExtra, + ForeignKeyTablesExtra, + FormHiddenArgsExtra, +] + + +table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) + + +async def resolve_table_extras(extras, context, include_internal=False): + return await table_extra_registry.resolve( + extras, context, ExtraScope.TABLE, include_internal=include_internal + ) + + +async def resolve_row_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.ROW) + + +async def resolve_query_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.QUERY) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75e4f3e8..19089dd1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,13 @@ Changelog ------------------- - Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) +- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. +- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. +- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. +- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. +- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +- Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. +- The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. .. _v1_0_a32: diff --git a/docs/json_api.rst b/docs/json_api.rst index 65031bf4..6b595577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -235,6 +235,831 @@ query string arguments: Only available if the :ref:`setting_trace_debug` setting is enabled. +.. _json_api_extra: + +Expanding JSON responses +------------------------ + +Table, row and query JSON responses can be expanded with one or more ``?_extra=`` parameters. +These can be repeated or comma-separated: + +:: + + ?_extra=columns&_extra=count,next_url + +.. [[[cog + from json_api_doc import table_extras + table_extras(cog) +.. ]]] + +Table JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The available table extras are listed below. + +``count`` + Total count of rows matching these filters (May execute additional queries.) + + ``GET /fixtures/facetable.json?_extra=count`` + + .. code-block:: json + + 15 + +``count_sql`` + SQL query used to calculate the total count + + ``GET /fixtures/facetable.json?_size=0&_extra=count_sql`` + + .. code-block:: json + + "select count(*) from facetable " + +``facet_results`` + Results of facets calculated against this data (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. + + .. code-block:: json + + { + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10 + }, + { + "value": "MI", + "label": "MI", + "count": 4 + } + ] + } + }, + "timed_out": [] + } + +``facets_timed_out`` + Facet calculations that timed out + + ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + + .. code-block:: json + + [] + +``suggested_facets`` + Suggestions for facets that might return interesting results (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. + + .. code-block:: json + + [ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state" + } + ] + +``human_description_en`` + Human-readable description of the filters + + ``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en`` + + .. code-block:: json + + "where state = \"CA\" sorted by pk" + +``next_url`` + Full URL for the next page of results + + ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + + .. code-block:: json + + "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" + +``columns`` + Column names returned by this query + + ``GET /fixtures/facetable.json?_extra=columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + + ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/facetable.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "pk" + ] + +``display_columns`` + Column metadata used by the HTML table display + + Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns. + + .. code-block:: json + + [ + { + "name": "pk", + "sortable": true, + "is_pk": true, + "type": "INTEGER", + "notnull": 0 + }, + { + "name": "created", + "sortable": true, + "is_pk": false, + "type": "TEXT", + "notnull": 0, + "description": null, + "column_type": null, + "column_type_config": null + } + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "id": 1, + "content": "hello" + }, + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + {}, + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/facetable.json?_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "table": "facetable", + "format": "json" + }, + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", + "nofacet": null, + "nosuggest": null + } + +``request`` + Full information about the request + + ``GET /fixtures/facetable.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/facetable.json?_extra=request", + "path": "/fixtures/facetable.json", + "full_path": "/fixtures/facetable.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/facetable.json?_size=1&_extra=query`` + + .. code-block:: json + + { + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2", + "params": {} + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``set_column_type_ui`` + Column type UI metadata for this table + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/facetable.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/facetable.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/facetable.json?_extra=table`` + + .. code-block:: json + + "facetable" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/facetable.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``renderers`` + Alternative output renderers available for this table + + ``GET /fixtures/facetable.json?_extra=renderers`` + + .. code-block:: json + + { + "json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on" + } + +``custom_table_templates`` + Custom template names considered for this table + + ``GET /fixtures/facetable.json?_extra=custom_table_templates`` + + .. code-block:: json + + [ + "_table-fixtures-facetable.html", + "_table-table-fixtures-facetable.html", + "_table.html" + ] + +``sorted_facet_results`` + Facet results sorted for display + + ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` + + .. code-block:: json + + [ + { + "name": "state", + "type": "column", + "hideable": true, + "toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA", + "selected": false + }, + { + "value": "MI", + "label": "MI", + "count": 4, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI", + "selected": false + }, + { + "value": "MC", + "label": "MC", + "count": 1, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC", + "selected": false + } + ], + "truncated": false + } + ] + +``table_definition`` + SQL definition for this table + + ``GET /fixtures/facetable.json?_extra=table_definition`` + + .. code-block:: json + + "CREATE TABLE facetable (\n pk integer primary key,\n created text,\n planet_int integer,\n on_earth integer,\n state text,\n _city_id integer,\n _neighborhood text,\n tags text,\n complex_array text,\n distinct_some_null,\n n text,\n FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);" + +``view_definition`` + SQL definition for this view + + ``GET /fixtures/simple_view.json?_extra=view_definition`` + + .. code-block:: json + + "CREATE VIEW simple_view AS\n SELECT content, upper(content) AS upper_content FROM simple_primary_key;" + +``is_view`` + Whether this resource is a view instead of a table + + ``GET /fixtures/simple_view.json?_extra=is_view`` + + .. code-block:: json + + true + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/facetable.json?_extra=private`` + + .. code-block:: json + + false + +``expandable_columns`` + Foreign key columns that can be expanded with labels + + ``GET /fixtures/facetable.json?_extra=expandable_columns`` + + .. code-block:: json + + [ + [ + { + "column": "_city_id", + "other_table": "facet_cities", + "other_column": "id" + }, + "name" + ] + ] + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + + ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args`` + + .. code-block:: json + + [ + [ + "_facet", + "state" + ], + [ + "_size", + "1" + ], + [ + "_extra", + "form_hidden_args" + ] + ] + +Row JSON responses +~~~~~~~~~~~~~~~~~~ + +The following extras are available for row JSON responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/simple_primary_key/1.json?_extra=columns`` + + .. code-block:: json + + [ + "id", + "content" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "id" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json" + }, + "resolved": { + "table": "simple_primary_key", + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + }, + "pks": [ + "id" + ], + "pk_values": [ + "1" + ] + } + } + +``request`` + Full information about the request + + ``GET /fixtures/simple_primary_key/1.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/simple_primary_key/1.json?_extra=request", + "path": "/fixtures/simple_primary_key/1.json", + "full_path": "/fixtures/simple_primary_key/1.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/simple_primary_key/1.json?_extra=query`` + + .. code-block:: json + + { + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + } + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/simple_primary_key/1.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/simple_primary_key/1.json?_extra=table`` + + .. code-block:: json + + "simple_primary_key" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/simple_primary_key/1.json?_extra=private`` + + .. code-block:: json + + false + +``foreign_key_tables`` + Tables that link to this row using foreign keys + + ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + + .. code-block:: json + + [ + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f1", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f1=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f2", + "count": 0, + "link": "/fixtures/complex_foreign_keys?f2=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f3", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f3=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_blank_label", + "count": 0, + "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_label", + "count": 1, + "link": "/fixtures/foreign_key_references?foreign_key_with_label=1" + } + ] + +Query JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The following extras are available for arbitrary SQL query responses and stored, named query responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns`` + + .. code-block:: json + + [ + "one" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "format": "json" + } + } + +``request`` + Full information about the request + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "path": "/fixtures/-/query.json", + "full_path": "/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "host": "localhost", + "args": { + "sql": [ + "select 1 as one" + ], + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query`` + + .. code-block:: json + + { + "sql": "select 1 as one", + "params": {} + } + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=query`` + + .. code-block:: json + + { + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "params": { + "text": "town" + } + } + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + + .. code-block:: json + + { + "database": "fixtures", + "name": "neighborhood_search", + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "title": "Search neighborhoods", + "description": null, + "description_html": null, + "hide_sql": false, + "fragment": null, + "params": [], + "parameters": [], + "is_write": false, + "is_private": false, + "is_trusted": true, + "owner_id": null, + "on_success_message": null, + "on_success_message_sql": null, + "on_success_redirect": null, + "on_error_message": null, + "on_error_redirect": null + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database`` + + .. code-block:: json + + "fixtures" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` + + .. code-block:: json + + false + +.. [[[end]]] + .. _table_arguments: Table arguments diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py new file mode 100644 index 00000000..44ef4a42 --- /dev/null +++ b/docs/json_api_doc.py @@ -0,0 +1,131 @@ +import asyncio +import json +import pathlib +import tempfile +import textwrap + + +def table_extras(cog): + from datasette.extras import ExtraScope + from datasette.views.table_extras import table_extra_registry + + scopes = [ + ( + ExtraScope.TABLE, + "Table JSON responses", + "The available table extras are listed below.", + ), + ( + ExtraScope.ROW, + "Row JSON responses", + "The following extras are available for row JSON responses.", + ), + ( + ExtraScope.QUERY, + "Query JSON responses", + ( + "The following extras are available for arbitrary SQL query " + "responses and stored, named query responses." + ), + ), + ] + classes_by_scope = [ + (scope, heading, intro, table_extra_registry.public_classes_for_scope(scope)) + for scope, heading, intro in scopes + ] + + live_examples = asyncio.run( + _fetch_live_examples( + [ + (scope, cls) + for scope, _, _, classes in classes_by_scope + for cls in classes + ] + ) + ) + cog.out("\n") + for scope, heading, intro, classes in classes_by_scope: + cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) + cog.out("{}\n\n".format(intro)) + for cls in classes: + examples = _examples_for_scope(cls, scope) + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() + + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + for example in examples: + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") + + +def _examples_for_scope(cls, scope): + examples = cls.example_for_scope(scope) + if examples is None: + return [] + if isinstance(examples, list): + return examples + return [examples] + + +async def _fetch_live_examples(scoped_classes): + from datasette.app import Datasette + from datasette.fixtures import write_fixture_database + + examples = {} + with tempfile.TemporaryDirectory() as tmpdir: + db_path = pathlib.Path(tmpdir) / "fixtures.db" + write_fixture_database(db_path) + datasette = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": textwrap.dedent(""" + select _neighborhood, facet_cities.name, state + from facetable + join facet_cities + on facetable._city_id = facet_cities.id + where _neighborhood like '%' || :text || '%' + order by _neighborhood; + """), + "title": "Search neighborhoods", + } + } + } + } + }, + ) + try: + for scope, cls in scoped_classes: + for example in _examples_for_scope(cls, scope): + if not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] + finally: + for db in datasette.databases.values(): + if not db.is_memory: + db.close() + return examples diff --git a/pyproject.toml b/pyproject.toml index 38085476..0d136d60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.6.1", + "asyncinject>=0.7", "setuptools", "pip", ] diff --git a/tests/test_api.py b/tests/test_api.py index f6187529..f57d0206 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -383,7 +383,7 @@ async def test_row_strange_table_name(ds_client): @pytest.mark.asyncio async def test_row_foreign_key_tables(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) assert response.status_code == 200 # Foreign keys are sorted by (other_table, column, other_column) @@ -426,6 +426,28 @@ async def test_row_foreign_key_tables(ds_client): ] +@pytest.mark.asyncio +async def test_row_extras(ds_client): + response = await ds_client.get( + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + ) + assert response.status_code == 200 + data = response.json() + assert data["database"] == "fixtures" + assert data["table"] == "simple_primary_key" + assert data["primary_keys"] == ["id"] + assert data["query"]["sql"] == 'select * from simple_primary_key where "id"=:p0' + assert data["query"]["params"] == {"p0": "1"} + assert data["request"]["path"] == "/fixtures/simple_primary_key/1.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json", + } + assert len(data["foreign_key_tables"]) == 5 + + @pytest.mark.asyncio async def test_row_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages""" diff --git a/tests/test_docs.py b/tests/test_docs.py index 51caf595..13b3a549 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -112,6 +112,51 @@ def test_table_filters_are_documented(documented_table_filters, subtests): assert f.key in documented_table_filters +def test_table_extra_examples_are_documented(): + from datasette.views.table_extras import CountExtra + + assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count" + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + assert "GET /fixtures/facetable.json?_extra=count" in section + assert ".. code-block:: json" in section + + +def test_render_cell_extra_example_explains_row_and_column_mapping(): + content = (docs_path / "json_api.rst").read_text() + section = content.split("``render_cell``")[-1].split("``query``")[0] + assert "same order as the ``rows`` array" in section + assert '"rows": [' in section + assert '"render_cell": [' in section + + +def test_debug_and_request_extra_examples_are_documented(): + content = (docs_path / "json_api.rst").read_text() + section = content.split("Table JSON responses")[-1].split("Row JSON responses")[0] + + debug_section = section.split("``debug``")[-1].split("``request``")[0] + assert "GET /fixtures/facetable.json?_extra=debug" in debug_section + assert '"url_vars": {' in debug_section + + request_section = section.split("``request``")[-1].split("``query``")[0] + assert "GET /fixtures/facetable.json?_extra=request" in request_section + assert '"full_path":' in request_section + + +def test_row_and_query_extra_sections_are_documented(): + content = (docs_path / "json_api.rst").read_text() + assert "Row JSON responses" in content + assert ( + "``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``" + in content + ) + assert "Query JSON responses" in content + assert "``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``" in content + assert ( + "``GET /fixtures/neighborhood_search.json?text=town&_extra=query``" in content + ) + + @pytest.fixture(scope="session") def documented_labels(): labels = set() diff --git a/tests/test_extras.py b/tests/test_extras.py new file mode 100644 index 00000000..ad8a9f00 --- /dev/null +++ b/tests/test_extras.py @@ -0,0 +1,65 @@ +import asyncio + +import pytest + +from datasette.extras import Extra, ExtraRegistry, ExtraScope + + +class SlowValueExtra(Extra): + description = "Returns context['value'], optionally slowly" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + if context["slow"]: + await asyncio.sleep(0.05) + return context["value"] + + +class DependentExtra(Extra): + description = "Depends on slow_value" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, slow_value): + return slow_value + 1 + + +def test_registry_is_built_once_per_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + first = registry._registry_for_scope(ExtraScope.TABLE) + second = registry._registry_for_scope(ExtraScope.TABLE) + assert first is second + + +@pytest.mark.asyncio +async def test_concurrent_resolves_do_not_share_state(): + # The asyncinject registry is shared across requests - resolved values + # must not leak between concurrent resolve() calls with different contexts + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + slow, fast = await asyncio.gather( + registry.resolve( + {"slow_value", "dependent"}, + {"value": 100, "slow": True}, + ExtraScope.TABLE, + ), + registry.resolve( + {"slow_value", "dependent"}, + {"value": 200, "slow": False}, + ExtraScope.TABLE, + ), + ) + assert slow == {"slow_value": 100, "dependent": 101} + assert fast == {"slow_value": 200, "dependent": 201} + + +@pytest.mark.asyncio +async def test_table_row_and_query_scopes_use_separate_registries(): + from datasette.views.table_extras import table_extra_registry + + registries = { + scope: table_extra_registry._registry_for_scope(scope) for scope in ExtraScope + } + assert len(set(map(id, registries.values()))) == 3 + # Scope-specific extras only registered where they belong + assert "count" in registries[ExtraScope.TABLE]._registry + assert "count" not in registries[ExtraScope.QUERY]._registry + assert "foreign_key_tables" in registries[ExtraScope.ROW]._registry diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ceeb646d..0cb67164 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -68,6 +68,134 @@ async def test_table_shape_arrayfirst(ds_client): ] +@pytest.mark.asyncio +async def test_query_extras_for_arbitrary_sql(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?" + + urllib.parse.urlencode( + { + "sql": "select 1 as one", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["rows"] == [{"one": 1}] + assert data["columns"] == ["one"] + assert data["database"] == "fixtures" + assert data["query"]["sql"] == "select 1 as one" + assert data["request"]["path"] == "/fixtures/-/query.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "format": "json", + } + + +@pytest.mark.asyncio +async def test_query_extras_for_stored_query(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?" + + urllib.parse.urlencode( + { + "text": "town", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["_neighborhood", "name", "state"] + assert data["database"] == "fixtures" + assert data["query"]["sql"].strip().startswith("select _neighborhood") + assert data["query"]["params"]["text"] == "town" + assert data["request"]["path"] == "/fixtures/neighborhood_search.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "neighborhood_search", + "format": "json", + } + + +@pytest.mark.parametrize("extra", ["filters", "actions", "display_rows"]) +@pytest.mark.asyncio +async def test_html_only_extras_are_not_available_via_json(ds_client, extra): + # These extras exist for the HTML view; their values are not JSON + # serializable so they are internal, not part of the JSON API + response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}") + assert response.status_code == 200 + assert extra not in response.json() + + +@pytest.mark.asyncio +async def test_html_only_extras_are_not_advertised(ds_client): + response = await ds_client.get("/fixtures/facetable.json?_extra=extras") + assert response.status_code == 200 + names = {e["name"] for e in response.json()["extras"]} + assert {"filters", "actions", "display_rows"}.isdisjoint(names) + + +def test_query_extra_private_for_arbitrary_sql(): + with make_app_client(config={"allow_sql": {"id": "root"}}) as client: + cookies = {"ds_actor": client.actor_cookie({"id": "root"})} + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private", + cookies=cookies, + ) + assert response.status == 200 + assert response.json["private"] is True + # Anonymous users cannot execute SQL at all here + anon = client.get("/fixtures/-/query.json?sql=select+1+as+one") + assert anon.status == 403 + + +def test_query_extra_query_reports_bound_params(): + config = { + "databases": { + "fixtures": { + "queries": { + "declared_params": { + "sql": "select 1 as one", + "params": ["foo"], + }, + "magic_host": { + "sql": "select :_header_host as h", + }, + } + } + } + } + with make_app_client(config=config) as client: + # Declared parameters are reported even when the regex cannot find them + response = client.get("/fixtures/declared_params.json?foo=bar&_extra=query") + assert response.status == 200 + assert response.json["query"]["params"] == {"foo": "bar"} + # Magic parameters are bound internally and should not be reported, + # especially not as a value taken from the querystring + response = client.get( + "/fixtures/magic_host.json?_extra=query&_header_host=spoofed" + ) + assert response.status == 200 + assert response.json["rows"] == [{"h": "localhost"}] + assert response.json["query"]["params"] == {} + + +def test_query_extra_query_does_not_echo_querystring_without_sql(): + with make_app_client() as client: + response = client.get("/fixtures/-/query.json?_extra=query&foo=bar") + assert response.status == 200 + assert response.json["query"]["params"] == {} + + +def test_query_extra_private_false_when_sql_is_public(): + with make_app_client() as client: + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ) + assert response.status == 200 + assert response.json["private"] is False + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") @@ -1376,6 +1504,17 @@ async def test_table_extras(ds_client, extra, expected_json): assert response.json() == expected_json +@pytest.mark.asyncio +async def test_table_extra_columns_can_be_comma_separated(ds_client): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=columns,count" + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["id", "content", "content2"] + assert data["count"] == 1 + + @pytest.mark.asyncio async def test_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook"""