Redesign and document extras mechanism to cover rows and queries in addition to tables

Merge PR #2769

datasette/_pytest_plugin.py

@@ -19,23 +19,38 @@ import weakref
 
 import pytest
 
-from datasette.app import Datasette
-
 _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar(
     "datasette_active_instances", default=None
 )
 
-_original_init = Datasette.__init__
+_original_init = None
 
 
-def _tracking_init(self, *args, **kwargs):
-    _original_init(self, *args, **kwargs)
-    instances = _active_instances.get()
-    if instances is not None:
-        instances.append(weakref.ref(self))
+def _install_tracking():
+    # datasette.app is imported lazily here rather than at module level:
+    # as a pytest11 entry point this module is imported during pytest
+    # startup, before pytest-cov starts measuring, so a module-level
+    # import would drag in all of datasette and make every import-time
+    # line in the package invisible to coverage
+    global _original_init
+    if _original_init is not None:
+        return
+    from datasette.app import Datasette
+
+    _original_init = Datasette.__init__
+
+    def _tracking_init(self, *args, **kwargs):
+        _original_init(self, *args, **kwargs)
+        instances = _active_instances.get()
+        if instances is not None:
+            instances.append(weakref.ref(self))
+
+    Datasette.__init__ = _tracking_init
 
 
-Datasette.__init__ = _tracking_init
+def pytest_configure(config):
+    if _enabled(config):
+        _install_tracking()
 
 
 def pytest_addoption(parser):

datasette/extras.py

@@ -0,0 +1,118 @@
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import ClassVar
+
+from asyncinject import Registry
+
+
+def extra_names_from_request(request):
+    extra_bits = request.args.getlist("_extra")
+    extras = set()
+    for bit in extra_bits:
+        extras.update(part for part in bit.split(",") if part)
+    return extras
+
+
+class ExtraScope(Enum):
+    TABLE = "table"
+    ROW = "row"
+    QUERY = "query"
+
+
+@dataclass(frozen=True)
+class ExtraExample:
+    path: str | None = None
+    key: str | None = None
+    value: object | None = None
+    note: str | None = None
+
+
+class Provider:
+    name: ClassVar[str | None] = None
+    scopes: ClassVar[set[ExtraScope]] = set()
+    public: ClassVar[bool] = False
+
+    @classmethod
+    def key(cls):
+        return cls.name or _camel_to_snake(cls.__name__)
+
+    @classmethod
+    def available_for(cls, scope):
+        return scope in cls.scopes
+
+    async def resolve(self, context):
+        raise NotImplementedError
+
+
+class Extra(Provider):
+    description: ClassVar[str | None] = None
+    example: ClassVar[ExtraExample | None] = None
+    examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {}
+    public: ClassVar[bool] = True
+    expensive: ClassVar[bool] = False
+    docs_note: ClassVar[str | None] = None
+
+    @classmethod
+    def example_for_scope(cls, scope):
+        return cls.examples.get(scope, cls.example)
+
+
+class ExtraRegistry:
+    def __init__(self, classes):
+        self.classes = list(classes)
+        self.classes_by_name = {cls.key(): cls for cls in self.classes}
+        # Lazily-built shared state, keyed by scope. Safe to share across
+        # requests because Extra instances are stateless and asyncinject's
+        # Registry keeps per-call state local to each resolve_multi() call.
+        # If extras classes ever become registerable at runtime (e.g. via a
+        # plugin hook) these caches will need invalidating.
+        self._scope_registries = {}
+        self._allowed_names = {}
+
+    def classes_for_scope(self, scope, include_internal=True):
+        classes = [
+            cls
+            for cls in self.classes
+            if cls.available_for(scope) and (include_internal or cls.public)
+        ]
+        return classes
+
+    def public_classes_for_scope(self, scope):
+        return self.classes_for_scope(scope, include_internal=False)
+
+    def _registry_for_scope(self, scope):
+        registry = self._scope_registries.get(scope)
+        if registry is None:
+            registry = Registry()
+            for cls in self.classes_for_scope(scope):
+                registry.register(cls().resolve, name=cls.key())
+            self._scope_registries[scope] = registry
+        return registry
+
+    def _allowed_names_for_scope(self, scope, include_internal):
+        key = (scope, include_internal)
+        names = self._allowed_names.get(key)
+        if names is None:
+            names = {
+                cls.key()
+                for cls in self.classes_for_scope(
+                    scope, include_internal=include_internal
+                )
+            }
+            self._allowed_names[key] = names
+        return names
+
+    async def resolve(self, requested, context, scope, include_internal=False):
+        allowed_names = self._allowed_names_for_scope(scope, include_internal)
+        requested_names = [name for name in requested if name in allowed_names]
+        resolved = await self._registry_for_scope(scope).resolve_multi(
+            requested_names, results={"context": context}
+        )
+        return {name: resolved[name] for name in requested_names}
+
+
+def _camel_to_snake(name):
+    name = re.sub(r"(Extra|Provider)$", "", name)
+    name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower()

datasette/renderer.py

@@ -1,4 +1,5 @@
 import json
+from datasette.extras import extra_names_from_request
 from datasette.utils import (
     value_as_boolean,
     remove_infinites,
@@ -108,7 +109,7 @@ def json_renderer(request, args, data, error, truncated=None):
 
     # Don't include "columns" in output
     # https://github.com/simonw/datasette/issues/2136
-    if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"):
+    if isinstance(data, dict) and "columns" not in extra_names_from_request(request):
         data.pop("columns", None)
 
     # Handle _nl option for _shape=array

datasette/views/database.py

@@ -11,6 +11,7 @@ import sqlite_utils
 import textwrap
 
 from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
+from datasette.extras import extra_names_from_request
 from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource, QueryResource
 from datasette.stored_queries import stored_query_to_dict
@@ -38,6 +39,11 @@ from datasette.plugins import pm
 
 from .base import BaseView, DatasetteError, View, _error, stream_csv
 from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
+from .table_extras import (
+    QueryExtraContext,
+    resolve_query_extras,
+    table_extra_registry,
+)
 from . import Context
 
 
@@ -606,11 +612,13 @@ class QueryView(View):
                 )
 
         else:
-            await datasette.ensure_permission(
+            visible, private = await datasette.check_visibility(
+                request.actor,
                 action="execute-sql",
                 resource=DatabaseResource(database=database),
-                actor=request.actor,
             )
+            if not visible:
+                raise Forbidden("execute-sql")
 
         # Flattened because of ?sql=&name1=value1&name2=value2 feature
         params = {key: request.args.get(key) for key in request.args}
@@ -692,6 +700,13 @@ class QueryView(View):
             except DatasetteError:
                 raise
 
+        async def query_metadata():
+            if stored_query:
+                metadata = stored_query_to_dict(stored_query)
+                metadata.pop("source", None)
+                return metadata
+            return await datasette.get_database_metadata(database)
+
         # Handle formats from plugins
         if format_ == "csv":
             if not sql:
@@ -704,6 +719,25 @@ class QueryView(View):
 
             return await stream_csv(datasette, fetch_data_for_csv, request, db.name)
         elif format_ in datasette.renderers.keys():
+            data = {"ok": True, "rows": rows, "columns": columns}
+            extras = extra_names_from_request(request)
+            if extras:
+                query_extra_context = QueryExtraContext(
+                    datasette=datasette,
+                    request=request,
+                    db=db,
+                    database_name=database,
+                    private=private,
+                    rows=rows,
+                    columns=columns,
+                    sql=sql,
+                    params=named_parameter_values,
+                    query_name=stored_query.name if stored_query else None,
+                    metadata=await query_metadata(),
+                    extras=extras,
+                    extra_registry=table_extra_registry,
+                )
+                data.update(await resolve_query_extras(extras, query_extra_context))
             # Dispatch request to the correct output format renderer
             # (CSV is not handled here due to streaming)
             result = call_with_supported_arguments(
@@ -721,7 +755,7 @@ class QueryView(View):
                 error=query_error,
                 # These will be deprecated in Datasette 1.0:
                 args=request.args,
-                data={"ok": True, "rows": rows, "columns": columns},
+                data=data,
             )
             if asyncio.iscoroutine(result):
                 result = await result
@@ -778,11 +812,7 @@ class QueryView(View):
                     )
                 }
             )
-            metadata = await datasette.get_database_metadata(database)
-            if stored_query:
-                metadata = stored_query_to_dict(stored_query)
-                metadata.pop("source", None)
-
+            metadata = await query_metadata()
             renderers = {}
             for key, (_, can_render) in datasette.renderers.items():
                 it_can_render = call_with_supported_arguments(

datasette/views/row.py

@@ -14,7 +14,9 @@ from datasette.plugins import pm
 import json
 import markupsafe
 import sqlite_utils
-from .table import display_columns_and_rows, _get_extras
+from datasette.extras import extra_names_from_request
+from .table import display_columns_and_rows
+from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry
 
 
 class RowView(DataView):
@@ -164,60 +166,27 @@ class RowView(DataView):
             "primary_key_values": pk_values,
         }
 
-        # Handle _extra parameter (new style)
-        extras = _get_extras(request)
-
-        # Also support legacy _extras parameter for backward compatibility
-        if "foreign_key_tables" in (request.args.get("_extras") or "").split(","):
-            extras.add("foreign_key_tables")
+        extras = extra_names_from_request(request)
 
         # Process extras
-        if "foreign_key_tables" in extras:
-            data["foreign_key_tables"] = await self.foreign_key_tables(
-                database, table, pk_values
-            )
-
-        if "render_cell" in extras:
-            # Call render_cell plugin hook for each cell
-            ct_map = await self.ds.get_column_types(database, table)
-            rendered_rows = []
-            for row in rows:
-                rendered_row = {}
-                for value, column in zip(row, columns):
-                    ct = ct_map.get(column)
-                    plugin_display_value = None
-                    # Try column type render_cell first
-                    if ct:
-                        candidate = await ct.render_cell(
-                            value=value,
-                            column=column,
-                            table=table,
-                            database=database,
-                            datasette=self.ds,
-                            request=request,
-                        )
-                        if candidate is not None:
-                            plugin_display_value = candidate
-                    if plugin_display_value is None:
-                        for candidate in pm.hook.render_cell(
-                            row=row,
-                            value=value,
-                            column=column,
-                            table=table,
-                            pks=resolved.pks,
-                            database=database,
-                            datasette=self.ds,
-                            request=request,
-                            column_type=ct,
-                        ):
-                            candidate = await await_me_maybe(candidate)
-                            if candidate is not None:
-                                plugin_display_value = candidate
-                                break
-                    if plugin_display_value:
-                        rendered_row[column] = str(plugin_display_value)
-                rendered_rows.append(rendered_row)
-            data["render_cell"] = rendered_rows
+        row_extra_context = RowExtraContext(
+            datasette=self.ds,
+            request=request,
+            db=db,
+            database_name=database,
+            table_name=table,
+            private=private,
+            rows=rows,
+            columns=columns,
+            pks=pks,
+            pk_values=pk_values,
+            sql=resolved.sql,
+            params=resolved.params,
+            extras=extras,
+            extra_registry=table_extra_registry,
+            foreign_key_tables=self.foreign_key_tables,
+        )
+        data.update(await resolve_row_extras(extras, row_extra_context))
 
         return (
             data,

datasette/views/table.py

@@ -3,11 +3,10 @@ import itertools
 import json
 import urllib
 
-from asyncinject import Registry
 import markupsafe
 
+from datasette.extras import extra_names_from_request
 from datasette.plugins import pm
-from datasette.database import QueryInterrupted
 from datasette.events import (
     AlterTableEvent,
     DropTableEvent,
@@ -46,6 +45,12 @@ from datasette.filters import Filters
 import sqlite_utils
 from .base import BaseView, DatasetteError, _error, stream_csv
 from .database import QueryView
+from .table_extras import (
+    TABLE_EXTRA_BUNDLES,
+    TableExtraContext,
+    resolve_table_extras,
+    table_extra_registry,
+)
 
 LINK_WITH_LABEL = (
     '<a href="{base_url}{database}/{table}/{link_id}">{label}</a>&nbsp;<em>{id}</em>'
@@ -849,14 +854,6 @@ class TableDropView(BaseView):
         return Response.json({"ok": True}, status=200)
 
 
-def _get_extras(request):
-    extra_bits = request.args.getlist("_extra")
-    extras = set()
-    for bit in extra_bits:
-        extras.update(bit.split(","))
-    return extras
-
-
 async def _columns_to_select(table_columns, pks, request):
     columns = list(table_columns)
     if "_col" in request.args:
@@ -1460,7 +1457,7 @@ async def table_view_data(
     rows = rows[:page_size]
 
     # Resolve extras
-    extras = _get_extras(request)
+    extras = extra_names_from_request(request)
     if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")):
         extras.add("facet_results")
     if request.args.get("_shape") == "object":
@@ -1468,559 +1465,65 @@ async def table_view_data(
     if extra_extras:
         extras.update(extra_extras)
 
-    async def extra_count_sql():
-        return count_sql
-
-    async def extra_count():
-        "Total count of rows matching these filters"
-        # Calculate the total count for this query
-        count = None
-        if (
-            not db.is_mutable
-            and datasette.inspect_data
-            and count_sql == f"select count(*) from {table_name} "
-        ):
-            # We can use a previously cached table row count
-            try:
-                count = datasette.inspect_data[database_name]["tables"][table_name][
-                    "count"
-                ]
-            except KeyError:
-                pass
-
-        # Otherwise run a select count(*) ...
-        if count_sql and count is None and not nocount:
-            count_sql_limited = (
-                f"select count(*) from (select * {from_sql} limit 10001)"
-            )
-            try:
-                count_rows = list(await db.execute(count_sql_limited, from_sql_params))
-                count = count_rows[0][0]
-            except QueryInterrupted:
-                pass
-        return count
-
-    async def facet_instances(extra_count):
-        facet_instances = []
-        facet_classes = list(
-            itertools.chain.from_iterable(pm.hook.register_facet_classes())
-        )
-        for facet_class in facet_classes:
-            facet_instances.append(
-                facet_class(
-                    datasette,
-                    request,
-                    database_name,
-                    sql=sql_no_order_no_limit,
-                    params=params,
-                    table=table_name,
-                    table_config=table_metadata,
-                    row_count=extra_count,
-                )
-            )
-        return facet_instances
-
-    async def extra_facet_results(facet_instances):
-        "Results of facets calculated against this data"
-        facet_results = {}
-        facets_timed_out = []
-
-        if not nofacet:
-            # Run them in parallel
-            facet_awaitables = [facet.facet_results() for facet in facet_instances]
-            facet_awaitable_results = await run_sequential(*facet_awaitables)
-            for (
-                instance_facet_results,
-                instance_facets_timed_out,
-            ) in facet_awaitable_results:
-                for facet_info in instance_facet_results:
-                    base_key = facet_info["name"]
-                    key = base_key
-                    i = 1
-                    while key in facet_results:
-                        i += 1
-                        key = f"{base_key}_{i}"
-                    facet_results[key] = facet_info
-                facets_timed_out.extend(instance_facets_timed_out)
-
-        return {
-            "results": facet_results,
-            "timed_out": facets_timed_out,
-        }
-
-    async def extra_suggested_facets(facet_instances):
-        "Suggestions for facets that might return interesting results"
-        suggested_facets = []
-        # Calculate suggested facets
-        if (
-            datasette.setting("suggest_facets")
-            and datasette.setting("allow_facet")
-            and not _next
-            and not nofacet
-            and not nosuggest
-        ):
-            # Run them in parallel
-            facet_suggest_awaitables = [facet.suggest() for facet in facet_instances]
-            for suggest_result in await run_sequential(*facet_suggest_awaitables):
-                suggested_facets.extend(suggest_result)
-        return suggested_facets
-
     # Faceting
     if not datasette.setting("allow_facet") and any(
         arg.startswith("_facet") for arg in request.args
     ):
         raise BadRequest("_facet= is not allowed")
 
-    # human_description_en combines filters AND search, if provided
-    async def extra_human_description_en():
-        "Human-readable description of the filters"
-        human_description_en = filters.human_description_en(
-            extra=extra_human_descriptions
-        )
-        if sort or sort_desc:
-            human_description_en = " ".join(
-                [b for b in [human_description_en, sorted_by] if b]
-            )
-        return human_description_en
-
-    if sort or sort_desc:
-        sorted_by = "sorted by {}{}".format(
-            (sort or sort_desc), " descending" if sort_desc else ""
-        )
-
-    async def extra_next_url():
-        "Full URL for the next page of results"
-        return next_url
-
-    async def extra_columns():
-        "Column names returned by this query"
-        return columns
-
-    async def extra_all_columns():
-        "All columns in the table, regardless of _col/_nocol filtering"
-        return list(table_columns)
-
-    async def extra_primary_keys():
-        "Primary keys for this table"
-        return pks
-
-    async def extra_actions():
-        async def actions():
-            links = []
-            kwargs = {
-                "datasette": datasette,
-                "database": database_name,
-                "actor": request.actor,
-                "request": request,
-            }
-            if is_view:
-                kwargs["view"] = table_name
-                method = pm.hook.view_actions
-            else:
-                kwargs["table"] = table_name
-                method = pm.hook.table_actions
-            for hook in method(**kwargs):
-                extra_links = await await_me_maybe(hook)
-                if extra_links:
-                    links.extend(extra_links)
-            return links
-
-        return actions
-
-    async def extra_is_view():
-        return is_view
-
-    async def extra_debug():
-        "Extra debug information"
-        return {
-            "resolved": repr(resolved),
-            "url_vars": request.url_vars,
-            "nofacet": nofacet,
-            "nosuggest": nosuggest,
-        }
-
-    async def extra_request():
-        "Full information about the request"
-        return {
-            "url": request.url,
-            "path": request.path,
-            "full_path": request.full_path,
-            "host": request.host,
-            "args": request.args._data,
-        }
-
-    async def run_display_columns_and_rows():
-        display_columns, display_rows = await display_columns_and_rows(
-            datasette,
-            database_name,
-            table_name,
-            results.description,
-            rows,
-            link_column=not is_view,
-            truncate_cells=datasette.setting("truncate_cells_html"),
-            sortable_columns=sortable_columns,
-            request=request,
-        )
-        return {
-            "columns": display_columns,
-            "rows": display_rows,
-        }
-
-    async def extra_display_columns(run_display_columns_and_rows):
-        return run_display_columns_and_rows["columns"]
-
-    async def extra_display_rows(run_display_columns_and_rows):
-        return run_display_columns_and_rows["rows"]
-
-    async def extra_render_cell():
-        "Rendered HTML for each cell using the render_cell plugin hook"
-        pks_for_display = pks if pks else (["rowid"] if not is_view else [])
-        col_names = [col[0] for col in results.description]
-        ct_map = await datasette.get_column_types(database_name, table_name)
-        rendered_rows = []
-        for row in rows:
-            rendered_row = {}
-            for value, column in zip(row, col_names):
-                ct = ct_map.get(column)
-                plugin_display_value = None
-                # Try column type render_cell first
-                if ct:
-                    candidate = await ct.render_cell(
-                        value=value,
-                        column=column,
-                        table=table_name,
-                        database=database_name,
-                        datasette=datasette,
-                        request=request,
-                    )
-                    if candidate is not None:
-                        plugin_display_value = candidate
-                if plugin_display_value is None:
-                    for candidate in pm.hook.render_cell(
-                        row=row,
-                        value=value,
-                        column=column,
-                        table=table_name,
-                        pks=pks_for_display,
-                        database=database_name,
-                        datasette=datasette,
-                        request=request,
-                        column_type=ct,
-                    ):
-                        candidate = await await_me_maybe(candidate)
-                        if candidate is not None:
-                            plugin_display_value = candidate
-                            break
-                if plugin_display_value:
-                    rendered_row[column] = str(plugin_display_value)
-            rendered_rows.append(rendered_row)
-        return rendered_rows
-
-    async def extra_query():
-        "Details of the underlying SQL query"
-        return {
-            "sql": sql,
-            "params": params,
-        }
-
-    async def extra_column_types():
-        "Column type assignments for this table"
-        ct_map = await datasette.get_column_types(database_name, table_name)
-        return {
-            col_name: {
-                "type": ct.name,
-                "config": ct.config,
-            }
-            for col_name, ct in ct_map.items()
-        }
-
-    async def extra_set_column_type_ui():
-        "Column type UI metadata for this table"
-        if is_view:
-            return None
-
-        if not await datasette.allowed(
-            action="set-column-type",
-            resource=TableResource(database=database_name, table=table_name),
-            actor=request.actor,
-        ):
-            return None
-
-        column_details = await datasette._get_resource_column_details(
-            database_name, table_name
-        )
-        ct_map = await datasette.get_column_types(database_name, table_name)
-        columns = {}
-        for column_name, column_detail in column_details.items():
-            current = ct_map.get(column_name)
-            columns[column_name] = {
-                "current": (
-                    {"type": current.name, "config": current.config}
-                    if current is not None
-                    else None
-                ),
-                "options": [
-                    {
-                        "name": name,
-                        "description": ct_cls.description,
-                    }
-                    for name, ct_cls in sorted(datasette._column_types.items())
-                    if datasette._column_type_is_applicable(ct_cls, column_detail)
-                ],
-            }
-        return {
-            "path": "{}/-/set-column-type".format(
-                datasette.urls.table(database_name, table_name)
-            ),
-            "columns": columns,
-        }
-
-    async def extra_metadata():
-        "Metadata about the table and database"
-        tablemetadata = await datasette.get_resource_metadata(database_name, table_name)
-
-        rows = await datasette.get_internal_database().execute(
-            """
-              SELECT
-                column_name,
-                value
-              FROM metadata_columns
-              WHERE database_name = ?
-                AND resource_name = ?
-                AND key = 'description'
-            """,
-            [database_name, table_name],
-        )
-        tablemetadata["columns"] = dict(rows)
-        return tablemetadata
-
-    async def extra_database():
-        return database_name
-
-    async def extra_table():
-        return table_name
-
-    async def extra_database_color():
-        return db.color
-
-    async def extra_form_hidden_args():
-        form_hidden_args = []
-        for key in request.args:
-            if (
-                key.startswith("_")
-                and key not in ("_sort", "_sort_desc", "_search", "_next")
-                and "__" not in key
-            ):
-                for value in request.args.getlist(key):
-                    form_hidden_args.append((key, value))
-        return form_hidden_args
-
-    async def extra_filters():
-        return filters
-
-    async def extra_custom_table_templates():
-        return [
-            f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html",
-            f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html",
-            "_table.html",
-        ]
-
-    async def extra_sorted_facet_results(extra_facet_results):
-        facet_configs = table_metadata.get("facets", [])
-        if facet_configs:
-            # Build ordered list of facet names from metadata config
-            metadata_facet_names = []
-            for fc in facet_configs:
-                if isinstance(fc, str):
-                    metadata_facet_names.append(fc)
-                elif isinstance(fc, dict):
-                    metadata_facet_names.append(list(fc.values())[0])
-            metadata_order = {name: i for i, name in enumerate(metadata_facet_names)}
-            metadata_facets = []
-            request_facets = []
-            for f in extra_facet_results["results"].values():
-                if f["name"] in metadata_order:
-                    metadata_facets.append(f)
-                else:
-                    request_facets.append(f)
-            metadata_facets.sort(key=lambda f: metadata_order[f["name"]])
-            request_facets.sort(
-                key=lambda f: (len(f["results"]), f["name"]),
-                reverse=True,
-            )
-            return metadata_facets + request_facets
-        else:
-            return sorted(
-                extra_facet_results["results"].values(),
-                key=lambda f: (len(f["results"]), f["name"]),
-                reverse=True,
-            )
-
-    async def extra_table_definition():
-        return await db.get_table_definition(table_name)
-
-    async def extra_view_definition():
-        return await db.get_view_definition(table_name)
-
-    async def extra_renderers(extra_expandable_columns, extra_query):
-        renderers = {}
-        url_labels_extra = {}
-        if extra_expandable_columns:
-            url_labels_extra = {"_labels": "on"}
-        for key, (_, can_render) in datasette.renderers.items():
-            it_can_render = call_with_supported_arguments(
-                can_render,
-                datasette=datasette,
-                columns=columns or [],
-                rows=rows or [],
-                sql=extra_query.get("sql", None),
-                query_name=None,
-                database=database_name,
-                table=table_name,
-                request=request,
-                view_name="table",
-            )
-            it_can_render = await await_me_maybe(it_can_render)
-            if it_can_render:
-                renderers[key] = datasette.urls.path(
-                    path_with_format(
-                        request=request,
-                        path=request.scope.get("route_path"),
-                        format=key,
-                        extra_qs={**url_labels_extra},
-                    )
-                )
-        return renderers
-
-    async def extra_private():
-        return private
-
-    async def extra_expandable_columns():
-        expandables = []
-        db = datasette.databases[database_name]
-        for fk in await db.foreign_keys_for_table(table_name):
-            label_column = await db.label_column_for_table(fk["other_table"])
-            expandables.append((fk, label_column))
-        return expandables
-
-    async def extra_extras():
-        "Available ?_extra= blocks"
-        all_extras = [
-            (key[len("extra_") :], fn.__doc__)
-            for key, fn in registry._registry.items()
-            if key.startswith("extra_")
-        ]
-        return [
-            {
-                "name": name,
-                "description": doc,
-                "toggle_url": datasette.absolute_url(
-                    request,
-                    datasette.urls.path(
-                        path_with_added_args(request, {"_extra": name})
-                        if name not in extras
-                        else path_with_removed_args(request, {"_extra": name})
-                    ),
-                ),
-                "selected": name in extras,
-            }
-            for name, doc in all_extras
-        ]
-
-    async def extra_facets_timed_out(extra_facet_results):
-        return extra_facet_results["timed_out"]
-
-    bundles = {
-        "html": [
-            "suggested_facets",
-            "facet_results",
-            "facets_timed_out",
-            "count",
-            "count_sql",
-            "human_description_en",
-            "next_url",
-            "metadata",
-            "query",
-            "columns",
-            "display_columns",
-            "display_rows",
-            "database",
-            "table",
-            "database_color",
-            "actions",
-            "filters",
-            "renderers",
-            "custom_table_templates",
-            "sorted_facet_results",
-            "table_definition",
-            "view_definition",
-            "is_view",
-            "private",
-            "primary_keys",
-            "all_columns",
-            "expandable_columns",
-            "form_hidden_args",
-            "set_column_type_ui",
-        ]
-    }
-
-    for key, values in bundles.items():
+    for key, values in TABLE_EXTRA_BUNDLES.items():
         if f"_{key}" in extras:
             extras.update(values)
         extras.discard(f"_{key}")
 
-    registry = Registry(
-        extra_count,
-        extra_count_sql,
-        extra_facet_results,
-        extra_facets_timed_out,
-        extra_suggested_facets,
-        facet_instances,
-        extra_human_description_en,
-        extra_next_url,
-        extra_columns,
-        extra_all_columns,
-        extra_primary_keys,
-        run_display_columns_and_rows,
-        extra_display_columns,
-        extra_display_rows,
-        extra_render_cell,
-        extra_debug,
-        extra_request,
-        extra_query,
-        extra_column_types,
-        extra_set_column_type_ui,
-        extra_metadata,
-        extra_extras,
-        extra_database,
-        extra_table,
-        extra_database_color,
-        extra_actions,
-        extra_filters,
-        extra_renderers,
-        extra_custom_table_templates,
-        extra_sorted_facet_results,
-        extra_table_definition,
-        extra_view_definition,
-        extra_is_view,
-        extra_private,
-        extra_expandable_columns,
-        extra_form_hidden_args,
+    table_extra_context = TableExtraContext(
+        datasette=datasette,
+        request=request,
+        resolved=resolved,
+        db=db,
+        database_name=database_name,
+        table_name=table_name,
+        is_view=is_view,
+        private=private,
+        rows=rows,
+        columns=columns,
+        results_description=results.description,
+        table_columns=table_columns,
+        pks=pks,
+        count_sql=count_sql,
+        from_sql=from_sql,
+        from_sql_params=from_sql_params,
+        nocount=nocount,
+        nofacet=nofacet,
+        nosuggest=nosuggest,
+        next_arg=request.args.get("_next"),
+        next_url=next_url,
+        sql=sql,
+        sql_no_order_no_limit=sql_no_order_no_limit,
+        params=params,
+        table_metadata=table_metadata,
+        filters=filters,
+        extra_human_descriptions=extra_human_descriptions,
+        sort=sort,
+        sort_desc=sort_desc,
+        sortable_columns=sortable_columns,
+        extras=extras,
+        extra_registry=table_extra_registry,
+        display_columns_and_rows=display_columns_and_rows,
+        run_sequential=run_sequential,
     )
 
-    results = await registry.resolve_multi(
-        ["extra_{}".format(extra) for extra in extras]
-    )
     data = {
         "ok": True,
         "next": next_value and str(next_value) or None,
     }
     data.update(
-        {
-            key.replace("extra_", ""): value
-            for key, value in results.items()
-            if key.startswith("extra_") and key.replace("extra_", "") in extras
-        }
+        await resolve_table_extras(
+            extras,
+            table_extra_context,
+            # The HTML view needs extras that are not JSON serializable
+            include_internal=bool(extra_extras),
+        )
     )
     raw_sqlite_rows = rows[:page_size]
     # Apply transform_value for columns with assigned types

docs/changelog.rst

@@ -10,6 +10,13 @@ Changelog
 -------------------
 
 - Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query <actions_update_query>` or :ref:`delete-query <actions_delete_query>` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`)
+- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list.
+- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`.
+- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response.
+- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``.
+- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring.
+- Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error.
+- The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead.
 
 .. _v1_0_a32:
 

docs/json_api.rst

@@ -235,6 +235,831 @@ query string arguments:
 
     Only available if the :ref:`setting_trace_debug` setting is enabled.
 
+.. _json_api_extra:
+
+Expanding JSON responses
+------------------------
+
+Table, row and query JSON responses can be expanded with one or more ``?_extra=`` parameters.
+These can be repeated or comma-separated:
+
+::
+
+    ?_extra=columns&_extra=count,next_url
+
+.. [[[cog
+    from json_api_doc import table_extras
+    table_extras(cog)
+.. ]]]
+
+Table JSON responses
+~~~~~~~~~~~~~~~~~~~~
+
+The available table extras are listed below.
+
+``count``
+    Total count of rows matching these filters (May execute additional queries.)
+
+    ``GET /fixtures/facetable.json?_extra=count``
+
+    .. code-block:: json
+
+        15
+
+``count_sql``
+    SQL query used to calculate the total count
+
+    ``GET /fixtures/facetable.json?_size=0&_extra=count_sql``
+
+    .. code-block:: json
+
+        "select count(*) from facetable "
+
+``facet_results``
+    Results of facets calculated against this data (May execute additional queries.)
+
+    Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.
+
+    .. code-block:: json
+
+        {
+          "results": {
+            "state": {
+              "name": "state",
+              "type": "column",
+              "results": [
+                {
+                  "value": "CA",
+                  "label": "CA",
+                  "count": 10
+                },
+                {
+                  "value": "MI",
+                  "label": "MI",
+                  "count": 4
+                }
+              ]
+            }
+          },
+          "timed_out": []
+        }
+
+``facets_timed_out``
+    Facet calculations that timed out
+
+    ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out``
+
+    .. code-block:: json
+
+        []
+
+``suggested_facets``
+    Suggestions for facets that might return interesting results (May execute additional queries.)
+
+    Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.
+
+    .. code-block:: json
+
+        [
+          {
+            "name": "state",
+            "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state"
+          }
+        ]
+
+``human_description_en``
+    Human-readable description of the filters
+
+    ``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en``
+
+    .. code-block:: json
+
+        "where state = \"CA\" sorted by pk"
+
+``next_url``
+    Full URL for the next page of results
+
+    ``GET /fixtures/facetable.json?_size=1&_extra=next_url``
+
+    .. code-block:: json
+
+        "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1"
+
+``columns``
+    Column names returned by this query
+
+    ``GET /fixtures/facetable.json?_extra=columns``
+
+    .. code-block:: json
+
+        [
+          "pk",
+          "created",
+          "planet_int",
+          "on_earth",
+          "state",
+          "_city_id",
+          "_neighborhood",
+          "tags",
+          "complex_array",
+          "distinct_some_null",
+          "n"
+        ]
+
+``all_columns``
+    All columns in the table, regardless of _col/_nocol filtering
+
+    ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns``
+
+    .. code-block:: json
+
+        [
+          "pk",
+          "created",
+          "planet_int",
+          "on_earth",
+          "state",
+          "_city_id",
+          "_neighborhood",
+          "tags",
+          "complex_array",
+          "distinct_some_null",
+          "n"
+        ]
+
+``primary_keys``
+    Primary keys for this table
+
+    ``GET /fixtures/facetable.json?_extra=primary_keys``
+
+    .. code-block:: json
+
+        [
+          "pk"
+        ]
+
+``display_columns``
+    Column metadata used by the HTML table display
+
+    Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.
+
+    .. code-block:: json
+
+        [
+          {
+            "name": "pk",
+            "sortable": true,
+            "is_pk": true,
+            "type": "INTEGER",
+            "notnull": 0
+          },
+          {
+            "name": "created",
+            "sortable": true,
+            "is_pk": false,
+            "type": "TEXT",
+            "notnull": 0,
+            "description": null,
+            "column_type": null,
+            "column_type_config": null
+          }
+        ]
+
+``render_cell``
+    Rendered HTML for each cell using the render_cell plugin hook
+
+    The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included.
+
+    .. code-block:: json
+
+        {
+          "rows": [
+            {
+              "id": 1,
+              "content": "hello"
+            },
+            {
+              "id": 4,
+              "content": "RENDER_CELL_DEMO"
+            }
+          ],
+          "render_cell": [
+            {},
+            {
+              "content": "<strong>Custom rendered HTML</strong>"
+            }
+          ]
+        }
+
+``debug``
+    Extra debug information
+
+    ``GET /fixtures/facetable.json?_extra=debug``
+
+    .. code-block:: json
+
+        {
+          "url_vars": {
+            "database": "fixtures",
+            "table": "facetable",
+            "format": "json"
+          },
+          "resolved": "ResolvedTable(db=<Database: fixtures (mutable, size=249856)>, table='facetable', is_view=False)",
+          "nofacet": null,
+          "nosuggest": null
+        }
+
+``request``
+    Full information about the request
+
+    ``GET /fixtures/facetable.json?_extra=request``
+
+    .. code-block:: json
+
+        {
+          "url": "http://localhost/fixtures/facetable.json?_extra=request",
+          "path": "/fixtures/facetable.json",
+          "full_path": "/fixtures/facetable.json?_extra=request",
+          "host": "localhost",
+          "args": {
+            "_extra": [
+              "request"
+            ]
+          }
+        }
+
+``query``
+    Details of the underlying SQL query
+
+    ``GET /fixtures/facetable.json?_size=1&_extra=query``
+
+    .. code-block:: json
+
+        {
+          "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2",
+          "params": {}
+        }
+
+``column_types``
+    Column type assignments for this table
+
+    .. code-block:: json
+
+        {}
+
+``set_column_type_ui``
+    Column type UI metadata for this table
+
+``metadata``
+    Metadata about the table, database or stored query
+
+    ``GET /fixtures/facetable.json?_extra=metadata``
+
+    .. code-block:: json
+
+        {
+          "columns": {}
+        }
+
+``extras``
+    Available ?_extra= blocks
+
+``database``
+    Database name
+
+    ``GET /fixtures/facetable.json?_extra=database``
+
+    .. code-block:: json
+
+        "fixtures"
+
+``table``
+    Table name
+
+    ``GET /fixtures/facetable.json?_extra=table``
+
+    .. code-block:: json
+
+        "facetable"
+
+``database_color``
+    Color assigned to the database
+
+    ``GET /fixtures/facetable.json?_extra=database_color``
+
+    .. code-block:: json
+
+        "9403e5"
+
+``renderers``
+    Alternative output renderers available for this table
+
+    ``GET /fixtures/facetable.json?_extra=renderers``
+
+    .. code-block:: json
+
+        {
+          "json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on"
+        }
+
+``custom_table_templates``
+    Custom template names considered for this table
+
+    ``GET /fixtures/facetable.json?_extra=custom_table_templates``
+
+    .. code-block:: json
+
+        [
+          "_table-fixtures-facetable.html",
+          "_table-table-fixtures-facetable.html",
+          "_table.html"
+        ]
+
+``sorted_facet_results``
+    Facet results sorted for display
+
+    ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results``
+
+    .. code-block:: json
+
+        [
+          {
+            "name": "state",
+            "type": "column",
+            "hideable": true,
+            "toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results",
+            "results": [
+              {
+                "value": "CA",
+                "label": "CA",
+                "count": 10,
+                "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA",
+                "selected": false
+              },
+              {
+                "value": "MI",
+                "label": "MI",
+                "count": 4,
+                "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI",
+                "selected": false
+              },
+              {
+                "value": "MC",
+                "label": "MC",
+                "count": 1,
+                "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC",
+                "selected": false
+              }
+            ],
+            "truncated": false
+          }
+        ]
+
+``table_definition``
+    SQL definition for this table
+
+    ``GET /fixtures/facetable.json?_extra=table_definition``
+
+    .. code-block:: json
+
+        "CREATE TABLE facetable (\n    pk integer primary key,\n    created text,\n    planet_int integer,\n    on_earth integer,\n    state text,\n    _city_id integer,\n    _neighborhood text,\n    tags text,\n    complex_array text,\n    distinct_some_null,\n    n text,\n    FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);"
+
+``view_definition``
+    SQL definition for this view
+
+    ``GET /fixtures/simple_view.json?_extra=view_definition``
+
+    .. code-block:: json
+
+        "CREATE VIEW simple_view AS\n    SELECT content, upper(content) AS upper_content FROM simple_primary_key;"
+
+``is_view``
+    Whether this resource is a view instead of a table
+
+    ``GET /fixtures/simple_view.json?_extra=is_view``
+
+    .. code-block:: json
+
+        true
+
+``private``
+    Whether this resource is private to the current actor
+
+    ``GET /fixtures/facetable.json?_extra=private``
+
+    .. code-block:: json
+
+        false
+
+``expandable_columns``
+    Foreign key columns that can be expanded with labels
+
+    ``GET /fixtures/facetable.json?_extra=expandable_columns``
+
+    .. code-block:: json
+
+        [
+          [
+            {
+              "column": "_city_id",
+              "other_table": "facet_cities",
+              "other_column": "id"
+            },
+            "name"
+          ]
+        ]
+
+``form_hidden_args``
+    Hidden form arguments used by the HTML table interface
+
+    ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args``
+
+    .. code-block:: json
+
+        [
+          [
+            "_facet",
+            "state"
+          ],
+          [
+            "_size",
+            "1"
+          ],
+          [
+            "_extra",
+            "form_hidden_args"
+          ]
+        ]
+
+Row JSON responses
+~~~~~~~~~~~~~~~~~~
+
+The following extras are available for row JSON responses.
+
+``columns``
+    Column names returned by this query
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=columns``
+
+    .. code-block:: json
+
+        [
+          "id",
+          "content"
+        ]
+
+``primary_keys``
+    Primary keys for this table
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys``
+
+    .. code-block:: json
+
+        [
+          "id"
+        ]
+
+``render_cell``
+    Rendered HTML for each cell using the render_cell plugin hook
+
+    The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included.
+
+    .. code-block:: json
+
+        {
+          "rows": [
+            {
+              "id": 4,
+              "content": "RENDER_CELL_DEMO"
+            }
+          ],
+          "render_cell": [
+            {
+              "content": "<strong>Custom rendered HTML</strong>"
+            }
+          ]
+        }
+
+``debug``
+    Extra debug information
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=debug``
+
+    .. code-block:: json
+
+        {
+          "url_vars": {
+            "database": "fixtures",
+            "table": "simple_primary_key",
+            "pks": "1",
+            "format": "json"
+          },
+          "resolved": {
+            "table": "simple_primary_key",
+            "sql": "select * from simple_primary_key where \"id\"=:p0",
+            "params": {
+              "p0": "1"
+            },
+            "pks": [
+              "id"
+            ],
+            "pk_values": [
+              "1"
+            ]
+          }
+        }
+
+``request``
+    Full information about the request
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=request``
+
+    .. code-block:: json
+
+        {
+          "url": "http://localhost/fixtures/simple_primary_key/1.json?_extra=request",
+          "path": "/fixtures/simple_primary_key/1.json",
+          "full_path": "/fixtures/simple_primary_key/1.json?_extra=request",
+          "host": "localhost",
+          "args": {
+            "_extra": [
+              "request"
+            ]
+          }
+        }
+
+``query``
+    Details of the underlying SQL query
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=query``
+
+    .. code-block:: json
+
+        {
+          "sql": "select * from simple_primary_key where \"id\"=:p0",
+          "params": {
+            "p0": "1"
+          }
+        }
+
+``column_types``
+    Column type assignments for this table
+
+    .. code-block:: json
+
+        {}
+
+``metadata``
+    Metadata about the table, database or stored query
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=metadata``
+
+    .. code-block:: json
+
+        {
+          "columns": {}
+        }
+
+``extras``
+    Available ?_extra= blocks
+
+``database``
+    Database name
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=database``
+
+    .. code-block:: json
+
+        "fixtures"
+
+``table``
+    Table name
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=table``
+
+    .. code-block:: json
+
+        "simple_primary_key"
+
+``database_color``
+    Color assigned to the database
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=database_color``
+
+    .. code-block:: json
+
+        "9403e5"
+
+``private``
+    Whether this resource is private to the current actor
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=private``
+
+    .. code-block:: json
+
+        false
+
+``foreign_key_tables``
+    Tables that link to this row using foreign keys
+
+    ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``
+
+    .. code-block:: json
+
+        [
+          {
+            "other_table": "complex_foreign_keys",
+            "column": "id",
+            "other_column": "f1",
+            "count": 1,
+            "link": "/fixtures/complex_foreign_keys?f1=1"
+          },
+          {
+            "other_table": "complex_foreign_keys",
+            "column": "id",
+            "other_column": "f2",
+            "count": 0,
+            "link": "/fixtures/complex_foreign_keys?f2=1"
+          },
+          {
+            "other_table": "complex_foreign_keys",
+            "column": "id",
+            "other_column": "f3",
+            "count": 1,
+            "link": "/fixtures/complex_foreign_keys?f3=1"
+          },
+          {
+            "other_table": "foreign_key_references",
+            "column": "id",
+            "other_column": "foreign_key_with_blank_label",
+            "count": 0,
+            "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1"
+          },
+          {
+            "other_table": "foreign_key_references",
+            "column": "id",
+            "other_column": "foreign_key_with_label",
+            "count": 1,
+            "link": "/fixtures/foreign_key_references?foreign_key_with_label=1"
+          }
+        ]
+
+Query JSON responses
+~~~~~~~~~~~~~~~~~~~~
+
+The following extras are available for arbitrary SQL query responses and stored, named query responses.
+
+``columns``
+    Column names returned by this query
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns``
+
+    .. code-block:: json
+
+        [
+          "one"
+        ]
+
+``render_cell``
+    Rendered HTML for each cell using the render_cell plugin hook
+
+    The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included.
+
+    .. code-block:: json
+
+        {
+          "rows": [
+            {
+              "content": "RENDER_CELL_DEMO"
+            }
+          ],
+          "render_cell": [
+            {
+              "content": "<strong>Custom rendered HTML</strong>"
+            }
+          ]
+        }
+
+``debug``
+    Extra debug information
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug``
+
+    .. code-block:: json
+
+        {
+          "url_vars": {
+            "database": "fixtures",
+            "format": "json"
+          }
+        }
+
+``request``
+    Full information about the request
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request``
+
+    .. code-block:: json
+
+        {
+          "url": "http://localhost/fixtures/-/query.json?sql=select+1+as+one&_extra=request",
+          "path": "/fixtures/-/query.json",
+          "full_path": "/fixtures/-/query.json?sql=select+1+as+one&_extra=request",
+          "host": "localhost",
+          "args": {
+            "sql": [
+              "select 1 as one"
+            ],
+            "_extra": [
+              "request"
+            ]
+          }
+        }
+
+``query``
+    Details of the underlying SQL query
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``
+
+    .. code-block:: json
+
+        {
+          "sql": "select 1 as one",
+          "params": {}
+        }
+
+    ``GET /fixtures/neighborhood_search.json?text=town&_extra=query``
+
+    .. code-block:: json
+
+        {
+          "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n    join facet_cities\n        on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n",
+          "params": {
+            "text": "town"
+          }
+        }
+
+``metadata``
+    Metadata about the table, database or stored query
+
+    ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata``
+
+    .. code-block:: json
+
+        {
+          "database": "fixtures",
+          "name": "neighborhood_search",
+          "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n    join facet_cities\n        on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n",
+          "title": "Search neighborhoods",
+          "description": null,
+          "description_html": null,
+          "hide_sql": false,
+          "fragment": null,
+          "params": [],
+          "parameters": [],
+          "is_write": false,
+          "is_private": false,
+          "is_trusted": true,
+          "owner_id": null,
+          "on_success_message": null,
+          "on_success_message_sql": null,
+          "on_success_redirect": null,
+          "on_error_message": null,
+          "on_error_redirect": null
+        }
+
+``extras``
+    Available ?_extra= blocks
+
+``database``
+    Database name
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database``
+
+    .. code-block:: json
+
+        "fixtures"
+
+``database_color``
+    Color assigned to the database
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color``
+
+    .. code-block:: json
+
+        "9403e5"
+
+``private``
+    Whether this resource is private to the current actor
+
+    ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private``
+
+    .. code-block:: json
+
+        false
+
+.. [[[end]]]
+
 .. _table_arguments:
 
 Table arguments

docs/json_api_doc.py

@@ -0,0 +1,131 @@
+import asyncio
+import json
+import pathlib
+import tempfile
+import textwrap
+
+
+def table_extras(cog):
+    from datasette.extras import ExtraScope
+    from datasette.views.table_extras import table_extra_registry
+
+    scopes = [
+        (
+            ExtraScope.TABLE,
+            "Table JSON responses",
+            "The available table extras are listed below.",
+        ),
+        (
+            ExtraScope.ROW,
+            "Row JSON responses",
+            "The following extras are available for row JSON responses.",
+        ),
+        (
+            ExtraScope.QUERY,
+            "Query JSON responses",
+            (
+                "The following extras are available for arbitrary SQL query "
+                "responses and stored, named query responses."
+            ),
+        ),
+    ]
+    classes_by_scope = [
+        (scope, heading, intro, table_extra_registry.public_classes_for_scope(scope))
+        for scope, heading, intro in scopes
+    ]
+
+    live_examples = asyncio.run(
+        _fetch_live_examples(
+            [
+                (scope, cls)
+                for scope, _, _, classes in classes_by_scope
+                for cls in classes
+            ]
+        )
+    )
+    cog.out("\n")
+    for scope, heading, intro, classes in classes_by_scope:
+        cog.out("{}\n{}\n\n".format(heading, "~" * len(heading)))
+        cog.out("{}\n\n".format(intro))
+        for cls in classes:
+            examples = _examples_for_scope(cls, scope)
+            description = cls.description or ""
+            notes = []
+            if cls.expensive:
+                notes.append("May execute additional queries.")
+            if cls.docs_note:
+                notes.append(cls.docs_note)
+            if notes:
+                description = "{} ({})".format(description, " ".join(notes)).strip()
+
+            cog.out("``{}``\n".format(cls.key()))
+            cog.out("    {}\n\n".format(description))
+            for example in examples:
+                if example.path:
+                    value = live_examples[(example.path, example.key or cls.key())]
+                    cog.out("    ``GET {}``\n\n".format(example.path))
+                else:
+                    value = example.value
+                if example.note:
+                    cog.out("    {}\n\n".format(example.note))
+                cog.out("    .. code-block:: json\n\n")
+                cog.out(textwrap.indent(json.dumps(value, indent=2), "        "))
+                cog.out("\n\n")
+
+
+def _examples_for_scope(cls, scope):
+    examples = cls.example_for_scope(scope)
+    if examples is None:
+        return []
+    if isinstance(examples, list):
+        return examples
+    return [examples]
+
+
+async def _fetch_live_examples(scoped_classes):
+    from datasette.app import Datasette
+    from datasette.fixtures import write_fixture_database
+
+    examples = {}
+    with tempfile.TemporaryDirectory() as tmpdir:
+        db_path = pathlib.Path(tmpdir) / "fixtures.db"
+        write_fixture_database(db_path)
+        datasette = Datasette(
+            [str(db_path)],
+            settings={"num_sql_threads": 1},
+            config={
+                "databases": {
+                    "fixtures": {
+                        "queries": {
+                            "neighborhood_search": {
+                                "sql": textwrap.dedent("""
+                                    select _neighborhood, facet_cities.name, state
+                                    from facetable
+                                        join facet_cities
+                                            on facetable._city_id = facet_cities.id
+                                    where _neighborhood like '%' || :text || '%'
+                                    order by _neighborhood;
+                                """),
+                                "title": "Search neighborhoods",
+                            }
+                        }
+                    }
+                }
+            },
+        )
+        try:
+            for scope, cls in scoped_classes:
+                for example in _examples_for_scope(cls, scope):
+                    if not example.path:
+                        continue
+                    key = example.key or cls.key()
+                    response = await datasette.client.get(example.path)
+                    assert response.status_code == 200, example.path
+                    data = response.json()
+                    assert key in data, "{} missing from {}".format(key, example.path)
+                    examples[(example.path, key)] = data[key]
+        finally:
+            for db in datasette.databases.values():
+                if not db.is_memory:
+                    db.close()
+    return examples

pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
     "mergedeep>=1.1.1",
     "itsdangerous>=1.1",
     "sqlite-utils>=3.30",
-    "asyncinject>=0.6.1",
+    "asyncinject>=0.7",
     "setuptools",
     "pip",
 ]

tests/test_api.py

@@ -383,7 +383,7 @@ async def test_row_strange_table_name(ds_client):
 @pytest.mark.asyncio
 async def test_row_foreign_key_tables(ds_client):
     response = await ds_client.get(
-        "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables"
+        "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables"
     )
     assert response.status_code == 200
     # Foreign keys are sorted by (other_table, column, other_column)
@@ -426,6 +426,28 @@ async def test_row_foreign_key_tables(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_row_extras(ds_client):
+    response = await ds_client.get(
+        "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables"
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["database"] == "fixtures"
+    assert data["table"] == "simple_primary_key"
+    assert data["primary_keys"] == ["id"]
+    assert data["query"]["sql"] == 'select * from simple_primary_key where "id"=:p0'
+    assert data["query"]["params"] == {"p0": "1"}
+    assert data["request"]["path"] == "/fixtures/simple_primary_key/1.json"
+    assert data["debug"]["url_vars"] == {
+        "database": "fixtures",
+        "table": "simple_primary_key",
+        "pks": "1",
+        "format": "json",
+    }
+    assert len(data["foreign_key_tables"]) == 5
+
+
 @pytest.mark.asyncio
 async def test_row_extra_render_cell():
     """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages"""

tests/test_docs.py

@@ -112,6 +112,51 @@ def test_table_filters_are_documented(documented_table_filters, subtests):
             assert f.key in documented_table_filters
 
 
+def test_table_extra_examples_are_documented():
+    from datasette.views.table_extras import CountExtra
+
+    assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count"
+    content = (docs_path / "json_api.rst").read_text()
+    section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0]
+    assert "GET /fixtures/facetable.json?_extra=count" in section
+    assert ".. code-block:: json" in section
+
+
+def test_render_cell_extra_example_explains_row_and_column_mapping():
+    content = (docs_path / "json_api.rst").read_text()
+    section = content.split("``render_cell``")[-1].split("``query``")[0]
+    assert "same order as the ``rows`` array" in section
+    assert '"rows": [' in section
+    assert '"render_cell": [' in section
+
+
+def test_debug_and_request_extra_examples_are_documented():
+    content = (docs_path / "json_api.rst").read_text()
+    section = content.split("Table JSON responses")[-1].split("Row JSON responses")[0]
+
+    debug_section = section.split("``debug``")[-1].split("``request``")[0]
+    assert "GET /fixtures/facetable.json?_extra=debug" in debug_section
+    assert '"url_vars": {' in debug_section
+
+    request_section = section.split("``request``")[-1].split("``query``")[0]
+    assert "GET /fixtures/facetable.json?_extra=request" in request_section
+    assert '"full_path":' in request_section
+
+
+def test_row_and_query_extra_sections_are_documented():
+    content = (docs_path / "json_api.rst").read_text()
+    assert "Row JSON responses" in content
+    assert (
+        "``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``"
+        in content
+    )
+    assert "Query JSON responses" in content
+    assert "``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``" in content
+    assert (
+        "``GET /fixtures/neighborhood_search.json?text=town&_extra=query``" in content
+    )
+
+
 @pytest.fixture(scope="session")
 def documented_labels():
     labels = set()

tests/test_extras.py

@@ -0,0 +1,65 @@
+import asyncio
+
+import pytest
+
+from datasette.extras import Extra, ExtraRegistry, ExtraScope
+
+
+class SlowValueExtra(Extra):
+    description = "Returns context['value'], optionally slowly"
+    scopes = {ExtraScope.TABLE}
+
+    async def resolve(self, context):
+        if context["slow"]:
+            await asyncio.sleep(0.05)
+        return context["value"]
+
+
+class DependentExtra(Extra):
+    description = "Depends on slow_value"
+    scopes = {ExtraScope.TABLE}
+
+    async def resolve(self, context, slow_value):
+        return slow_value + 1
+
+
+def test_registry_is_built_once_per_scope():
+    registry = ExtraRegistry([SlowValueExtra, DependentExtra])
+    first = registry._registry_for_scope(ExtraScope.TABLE)
+    second = registry._registry_for_scope(ExtraScope.TABLE)
+    assert first is second
+
+
+@pytest.mark.asyncio
+async def test_concurrent_resolves_do_not_share_state():
+    # The asyncinject registry is shared across requests - resolved values
+    # must not leak between concurrent resolve() calls with different contexts
+    registry = ExtraRegistry([SlowValueExtra, DependentExtra])
+    slow, fast = await asyncio.gather(
+        registry.resolve(
+            {"slow_value", "dependent"},
+            {"value": 100, "slow": True},
+            ExtraScope.TABLE,
+        ),
+        registry.resolve(
+            {"slow_value", "dependent"},
+            {"value": 200, "slow": False},
+            ExtraScope.TABLE,
+        ),
+    )
+    assert slow == {"slow_value": 100, "dependent": 101}
+    assert fast == {"slow_value": 200, "dependent": 201}
+
+
+@pytest.mark.asyncio
+async def test_table_row_and_query_scopes_use_separate_registries():
+    from datasette.views.table_extras import table_extra_registry
+
+    registries = {
+        scope: table_extra_registry._registry_for_scope(scope) for scope in ExtraScope
+    }
+    assert len(set(map(id, registries.values()))) == 3
+    # Scope-specific extras only registered where they belong
+    assert "count" in registries[ExtraScope.TABLE]._registry
+    assert "count" not in registries[ExtraScope.QUERY]._registry
+    assert "foreign_key_tables" in registries[ExtraScope.ROW]._registry

tests/test_table_api.py

@@ -68,6 +68,134 @@ async def test_table_shape_arrayfirst(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_query_extras_for_arbitrary_sql(ds_client):
+    response = await ds_client.get(
+        "/fixtures/-/query.json?"
+        + urllib.parse.urlencode(
+            {
+                "sql": "select 1 as one",
+                "_extra": "columns,database,query,request,debug",
+            }
+        )
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["rows"] == [{"one": 1}]
+    assert data["columns"] == ["one"]
+    assert data["database"] == "fixtures"
+    assert data["query"]["sql"] == "select 1 as one"
+    assert data["request"]["path"] == "/fixtures/-/query.json"
+    assert data["debug"]["url_vars"] == {
+        "database": "fixtures",
+        "format": "json",
+    }
+
+
+@pytest.mark.asyncio
+async def test_query_extras_for_stored_query(ds_client):
+    response = await ds_client.get(
+        "/fixtures/neighborhood_search.json?"
+        + urllib.parse.urlencode(
+            {
+                "text": "town",
+                "_extra": "columns,database,query,request,debug",
+            }
+        )
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["columns"] == ["_neighborhood", "name", "state"]
+    assert data["database"] == "fixtures"
+    assert data["query"]["sql"].strip().startswith("select _neighborhood")
+    assert data["query"]["params"]["text"] == "town"
+    assert data["request"]["path"] == "/fixtures/neighborhood_search.json"
+    assert data["debug"]["url_vars"] == {
+        "database": "fixtures",
+        "table": "neighborhood_search",
+        "format": "json",
+    }
+
+
+@pytest.mark.parametrize("extra", ["filters", "actions", "display_rows"])
+@pytest.mark.asyncio
+async def test_html_only_extras_are_not_available_via_json(ds_client, extra):
+    # These extras exist for the HTML view; their values are not JSON
+    # serializable so they are internal, not part of the JSON API
+    response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}")
+    assert response.status_code == 200
+    assert extra not in response.json()
+
+
+@pytest.mark.asyncio
+async def test_html_only_extras_are_not_advertised(ds_client):
+    response = await ds_client.get("/fixtures/facetable.json?_extra=extras")
+    assert response.status_code == 200
+    names = {e["name"] for e in response.json()["extras"]}
+    assert {"filters", "actions", "display_rows"}.isdisjoint(names)
+
+
+def test_query_extra_private_for_arbitrary_sql():
+    with make_app_client(config={"allow_sql": {"id": "root"}}) as client:
+        cookies = {"ds_actor": client.actor_cookie({"id": "root"})}
+        response = client.get(
+            "/fixtures/-/query.json?sql=select+1+as+one&_extra=private",
+            cookies=cookies,
+        )
+        assert response.status == 200
+        assert response.json["private"] is True
+        # Anonymous users cannot execute SQL at all here
+        anon = client.get("/fixtures/-/query.json?sql=select+1+as+one")
+        assert anon.status == 403
+
+
+def test_query_extra_query_reports_bound_params():
+    config = {
+        "databases": {
+            "fixtures": {
+                "queries": {
+                    "declared_params": {
+                        "sql": "select 1 as one",
+                        "params": ["foo"],
+                    },
+                    "magic_host": {
+                        "sql": "select :_header_host as h",
+                    },
+                }
+            }
+        }
+    }
+    with make_app_client(config=config) as client:
+        # Declared parameters are reported even when the regex cannot find them
+        response = client.get("/fixtures/declared_params.json?foo=bar&_extra=query")
+        assert response.status == 200
+        assert response.json["query"]["params"] == {"foo": "bar"}
+        # Magic parameters are bound internally and should not be reported,
+        # especially not as a value taken from the querystring
+        response = client.get(
+            "/fixtures/magic_host.json?_extra=query&_header_host=spoofed"
+        )
+        assert response.status == 200
+        assert response.json["rows"] == [{"h": "localhost"}]
+        assert response.json["query"]["params"] == {}
+
+
+def test_query_extra_query_does_not_echo_querystring_without_sql():
+    with make_app_client() as client:
+        response = client.get("/fixtures/-/query.json?_extra=query&foo=bar")
+        assert response.status == 200
+        assert response.json["query"]["params"] == {}
+
+
+def test_query_extra_private_false_when_sql_is_public():
+    with make_app_client() as client:
+        response = client.get(
+            "/fixtures/-/query.json?sql=select+1+as+one&_extra=private"
+        )
+        assert response.status == 200
+        assert response.json["private"] is False
+
+
 @pytest.mark.asyncio
 async def test_table_shape_objects(ds_client):
     response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects")
@@ -1376,6 +1504,17 @@ async def test_table_extras(ds_client, extra, expected_json):
     assert response.json() == expected_json
 
 
+@pytest.mark.asyncio
+async def test_table_extra_columns_can_be_comma_separated(ds_client):
+    response = await ds_client.get(
+        "/fixtures/primary_key_multiple_columns.json?_extra=columns,count"
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["columns"] == ["id", "content", "content2"]
+    assert data["count"] == 1
+
+
 @pytest.mark.asyncio
 async def test_extra_render_cell():
     """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook"""