Add generated examples for table JSON extras

This commit is contained in:
Simon Willison 2026-06-08 21:10:58 -07:00
commit 79c8aff31d
5 changed files with 562 additions and 82 deletions

View file

@ -1,4 +1,5 @@
import re
from dataclasses import dataclass
from enum import Enum
from typing import ClassVar
@ -17,6 +18,14 @@ class ExtraScope(Enum):
TABLE = "table"
@dataclass(frozen=True)
class ExtraExample:
path: str | None = None
key: str | None = None
value: object | None = None
note: str | None = None
class Provider:
name: ClassVar[str | None] = None
scopes: ClassVar[frozenset[ExtraScope]] = frozenset()
@ -36,6 +45,7 @@ class Provider:
class Extra(Provider):
description: ClassVar[str | None] = None
example: ClassVar[ExtraExample | None] = None
public: ClassVar[bool] = True
stable: ClassVar[bool] = True
expensive: ClassVar[bool] = False
@ -52,6 +62,7 @@ class Extra(Provider):
"stable": cls.stable,
"expensive": cls.expensive,
"docs_note": cls.docs_note,
"example": cls.example,
}

View file

@ -2,7 +2,7 @@ import itertools
from dataclasses import dataclass
from datasette.database import QueryInterrupted
from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider
from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider
from datasette.plugins import pm
from datasette.resources import TableResource
from datasette.utils import (
@ -56,6 +56,7 @@ class TableExtraContext:
class CountSqlExtra(Extra):
description = "SQL query used to calculate the total count"
example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -64,6 +65,7 @@ class CountSqlExtra(Extra):
class CountExtra(Extra):
description = "Total count of rows matching these filters"
example = ExtraExample("/fixtures/facetable.json?_extra=count")
scopes = frozenset({ExtraScope.TABLE})
expensive = True
@ -121,6 +123,22 @@ class FacetInstancesProvider(Provider):
class FacetResultsExtra(Extra):
description = "Results of facets calculated against this data"
example = ExtraExample(
value={
"results": {
"state": {
"name": "state",
"type": "column",
"results": [
{"value": "CA", "label": "CA", "count": 10},
{"value": "MI", "label": "MI", "count": 4},
],
}
},
"timed_out": [],
},
note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.",
)
scopes = frozenset({ExtraScope.TABLE})
expensive = True
@ -153,6 +171,9 @@ class FacetResultsExtra(Extra):
class FacetsTimedOutExtra(Extra):
description = "Facet calculations that timed out"
example = ExtraExample(
"/fixtures/facetable.json?_facet=state&_extra=facets_timed_out"
)
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context, facet_results):
@ -161,6 +182,15 @@ class FacetsTimedOutExtra(Extra):
class SuggestedFacetsExtra(Extra):
description = "Suggestions for facets that might return interesting results"
example = ExtraExample(
value=[
{
"name": "state",
"toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state",
}
],
note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.",
)
scopes = frozenset({ExtraScope.TABLE})
expensive = True
@ -183,6 +213,9 @@ class SuggestedFacetsExtra(Extra):
class HumanDescriptionEnExtra(Extra):
description = "Human-readable description of the filters"
example = ExtraExample(
"/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en"
)
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -202,6 +235,7 @@ class HumanDescriptionEnExtra(Extra):
class NextUrlExtra(Extra):
description = "Full URL for the next page of results"
example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -210,6 +244,7 @@ class NextUrlExtra(Extra):
class ColumnsExtra(Extra):
description = "Column names returned by this query"
example = ExtraExample("/fixtures/facetable.json?_extra=columns")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -218,6 +253,7 @@ class ColumnsExtra(Extra):
class AllColumnsExtra(Extra):
description = "All columns in the table, regardless of _col/_nocol filtering"
example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -226,6 +262,7 @@ class AllColumnsExtra(Extra):
class PrimaryKeysExtra(Extra):
description = "Primary keys for this table"
example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -262,6 +299,7 @@ class ActionsExtra(Extra):
class IsViewExtra(Extra):
description = "Whether this resource is a view instead of a table"
example = ExtraExample("/fixtures/simple_view.json?_extra=is_view")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -318,6 +356,28 @@ class DisplayColumnsAndRowsProvider(Provider):
class DisplayColumnsExtra(Extra):
description = "Column metadata used by the HTML table display"
example = ExtraExample(
value=[
{
"name": "pk",
"sortable": True,
"is_pk": True,
"type": "INTEGER",
"notnull": 0,
},
{
"name": "created",
"sortable": True,
"is_pk": False,
"type": "TEXT",
"notnull": 0,
"description": None,
"column_type": None,
"column_type_config": None,
},
],
note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.",
)
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context, display_columns_and_rows):
@ -334,6 +394,13 @@ class DisplayRowsExtra(Extra):
class RenderCellExtra(Extra):
description = "Rendered HTML for each cell using the render_cell plugin hook"
example = ExtraExample(
value=[
{},
{"content": "<strong>Custom rendered HTML</strong>"},
],
note="Only columns whose rendered value differs from the default are included.",
)
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -385,6 +452,7 @@ class RenderCellExtra(Extra):
class QueryExtra(Extra):
description = "Details of the underlying SQL query"
example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -396,6 +464,7 @@ class QueryExtra(Extra):
class ColumnTypesExtra(Extra):
description = "Column type assignments for this table"
example = ExtraExample(value={})
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -464,6 +533,7 @@ class SetColumnTypeUiExtra(Extra):
class MetadataExtra(Extra):
description = "Metadata about the table and database"
example = ExtraExample("/fixtures/facetable.json?_extra=metadata")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -489,6 +559,7 @@ class MetadataExtra(Extra):
class DatabaseExtra(Extra):
description = "Database name"
example = ExtraExample("/fixtures/facetable.json?_extra=database")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -497,6 +568,7 @@ class DatabaseExtra(Extra):
class TableExtra(Extra):
description = "Table name"
example = ExtraExample("/fixtures/facetable.json?_extra=table")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -505,6 +577,7 @@ class TableExtra(Extra):
class DatabaseColorExtra(Extra):
description = "Color assigned to the database"
example = ExtraExample("/fixtures/facetable.json?_extra=database_color")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -513,6 +586,9 @@ class DatabaseColorExtra(Extra):
class FormHiddenArgsExtra(Extra):
description = "Hidden form arguments used by the HTML table interface"
example = ExtraExample(
"/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args"
)
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -538,6 +614,7 @@ class FiltersExtra(Extra):
class CustomTableTemplatesExtra(Extra):
description = "Custom template names considered for this table"
example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -550,6 +627,9 @@ class CustomTableTemplatesExtra(Extra):
class SortedFacetResultsExtra(Extra):
description = "Facet results sorted for display"
example = ExtraExample(
"/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results"
)
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context, facet_results):
@ -585,6 +665,7 @@ class SortedFacetResultsExtra(Extra):
class TableDefinitionExtra(Extra):
description = "SQL definition for this table"
example = ExtraExample("/fixtures/facetable.json?_extra=table_definition")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -593,6 +674,7 @@ class TableDefinitionExtra(Extra):
class ViewDefinitionExtra(Extra):
description = "SQL definition for this view"
example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -601,6 +683,7 @@ class ViewDefinitionExtra(Extra):
class RenderersExtra(Extra):
description = "Alternative output renderers available for this table"
example = ExtraExample("/fixtures/facetable.json?_extra=renderers")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context, expandable_columns, query):
@ -636,6 +719,7 @@ class RenderersExtra(Extra):
class PrivateExtra(Extra):
description = "Whether this table is private to the current actor"
example = ExtraExample("/fixtures/facetable.json?_extra=private")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):
@ -644,6 +728,7 @@ class PrivateExtra(Extra):
class ExpandableColumnsExtra(Extra):
description = "Foreign key columns that can be expanded with labels"
example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns")
scopes = frozenset({ExtraScope.TABLE})
async def resolve(self, context):

View file

@ -254,79 +254,405 @@ The available table extras are listed below.
table_extras(cog)
.. ]]]
.. list-table::
:header-rows: 1
``count``
Total count of rows matching these filters (May execute additional queries.)
* - Extra
- Description
* - ``count``
- Total count of rows matching these filters (May execute additional queries.)
* - ``count_sql``
- SQL query used to calculate the total count
* - ``facet_results``
- Results of facets calculated against this data (May execute additional queries.)
* - ``facets_timed_out``
- Facet calculations that timed out
* - ``suggested_facets``
- Suggestions for facets that might return interesting results (May execute additional queries.)
* - ``human_description_en``
- Human-readable description of the filters
* - ``next_url``
- Full URL for the next page of results
* - ``columns``
- Column names returned by this query
* - ``all_columns``
- All columns in the table, regardless of _col/_nocol filtering
* - ``primary_keys``
- Primary keys for this table
* - ``display_columns``
- Column metadata used by the HTML table display
* - ``display_rows``
- Row data formatted for the HTML table display
* - ``render_cell``
- Rendered HTML for each cell using the render_cell plugin hook
* - ``debug``
- Extra debug information
* - ``request``
- Full information about the request
* - ``query``
- Details of the underlying SQL query
* - ``column_types``
- Column type assignments for this table
* - ``set_column_type_ui``
- Column type UI metadata for this table
* - ``metadata``
- Metadata about the table and database
* - ``extras``
- Available ?_extra= blocks
* - ``database``
- Database name
* - ``table``
- Table name
* - ``database_color``
- Color assigned to the database
* - ``actions``
- Table or view actions made available by plugin hooks
* - ``filters``
- Filters object used by the HTML table interface
* - ``renderers``
- Alternative output renderers available for this table
* - ``custom_table_templates``
- Custom template names considered for this table
* - ``sorted_facet_results``
- Facet results sorted for display
* - ``table_definition``
- SQL definition for this table
* - ``view_definition``
- SQL definition for this view
* - ``is_view``
- Whether this resource is a view instead of a table
* - ``private``
- Whether this table is private to the current actor
* - ``expandable_columns``
- Foreign key columns that can be expanded with labels
* - ``form_hidden_args``
- Hidden form arguments used by the HTML table interface
``GET /fixtures/facetable.json?_extra=count``
.. code-block:: json
15
``count_sql``
SQL query used to calculate the total count
``GET /fixtures/facetable.json?_size=0&_extra=count_sql``
.. code-block:: json
"select count(*) from facetable "
``facet_results``
Results of facets calculated against this data (May execute additional queries.)
Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.
.. code-block:: json
{
"results": {
"state": {
"name": "state",
"type": "column",
"results": [
{
"value": "CA",
"label": "CA",
"count": 10
},
{
"value": "MI",
"label": "MI",
"count": 4
}
]
}
},
"timed_out": []
}
``facets_timed_out``
Facet calculations that timed out
``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out``
.. code-block:: json
[]
``suggested_facets``
Suggestions for facets that might return interesting results (May execute additional queries.)
Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.
.. code-block:: json
[
{
"name": "state",
"toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state"
}
]
``human_description_en``
Human-readable description of the filters
``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en``
.. code-block:: json
"where state = \"CA\" sorted by pk"
``next_url``
Full URL for the next page of results
``GET /fixtures/facetable.json?_size=1&_extra=next_url``
.. code-block:: json
"http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1"
``columns``
Column names returned by this query
``GET /fixtures/facetable.json?_extra=columns``
.. code-block:: json
[
"pk",
"created",
"planet_int",
"on_earth",
"state",
"_city_id",
"_neighborhood",
"tags",
"complex_array",
"distinct_some_null",
"n"
]
``all_columns``
All columns in the table, regardless of _col/_nocol filtering
``GET /fixtures/facetable.json?_col=pk&_extra=all_columns``
.. code-block:: json
[
"pk",
"created",
"planet_int",
"on_earth",
"state",
"_city_id",
"_neighborhood",
"tags",
"complex_array",
"distinct_some_null",
"n"
]
``primary_keys``
Primary keys for this table
``GET /fixtures/facetable.json?_extra=primary_keys``
.. code-block:: json
[
"pk"
]
``display_columns``
Column metadata used by the HTML table display
Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.
.. code-block:: json
[
{
"name": "pk",
"sortable": true,
"is_pk": true,
"type": "INTEGER",
"notnull": 0
},
{
"name": "created",
"sortable": true,
"is_pk": false,
"type": "TEXT",
"notnull": 0,
"description": null,
"column_type": null,
"column_type_config": null
}
]
``display_rows``
Row data formatted for the HTML table display
``render_cell``
Rendered HTML for each cell using the render_cell plugin hook
Only columns whose rendered value differs from the default are included.
.. code-block:: json
[
{},
{
"content": "<strong>Custom rendered HTML</strong>"
}
]
``debug``
Extra debug information
``request``
Full information about the request
``query``
Details of the underlying SQL query
``GET /fixtures/facetable.json?_size=1&_extra=query``
.. code-block:: json
{
"sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2",
"params": {}
}
``column_types``
Column type assignments for this table
.. code-block:: json
{}
``set_column_type_ui``
Column type UI metadata for this table
``metadata``
Metadata about the table and database
``GET /fixtures/facetable.json?_extra=metadata``
.. code-block:: json
{
"columns": {}
}
``extras``
Available ?_extra= blocks
``database``
Database name
``GET /fixtures/facetable.json?_extra=database``
.. code-block:: json
"fixtures"
``table``
Table name
``GET /fixtures/facetable.json?_extra=table``
.. code-block:: json
"facetable"
``database_color``
Color assigned to the database
``GET /fixtures/facetable.json?_extra=database_color``
.. code-block:: json
"9403e5"
``actions``
Table or view actions made available by plugin hooks
``filters``
Filters object used by the HTML table interface
``renderers``
Alternative output renderers available for this table
``GET /fixtures/facetable.json?_extra=renderers``
.. code-block:: json
{
"json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on"
}
``custom_table_templates``
Custom template names considered for this table
``GET /fixtures/facetable.json?_extra=custom_table_templates``
.. code-block:: json
[
"_table-fixtures-facetable.html",
"_table-table-fixtures-facetable.html",
"_table.html"
]
``sorted_facet_results``
Facet results sorted for display
``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results``
.. code-block:: json
[
{
"name": "state",
"type": "column",
"hideable": true,
"toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results",
"results": [
{
"value": "CA",
"label": "CA",
"count": 10,
"toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA",
"selected": false
},
{
"value": "MI",
"label": "MI",
"count": 4,
"toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI",
"selected": false
},
{
"value": "MC",
"label": "MC",
"count": 1,
"toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC",
"selected": false
}
],
"truncated": false
}
]
``table_definition``
SQL definition for this table
``GET /fixtures/facetable.json?_extra=table_definition``
.. code-block:: json
"CREATE TABLE facetable (\n pk integer primary key,\n created text,\n planet_int integer,\n on_earth integer,\n state text,\n _city_id integer,\n _neighborhood text,\n tags text,\n complex_array text,\n distinct_some_null,\n n text,\n FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);"
``view_definition``
SQL definition for this view
``GET /fixtures/simple_view.json?_extra=view_definition``
.. code-block:: json
"CREATE VIEW simple_view AS\n SELECT content, upper(content) AS upper_content FROM simple_primary_key;"
``is_view``
Whether this resource is a view instead of a table
``GET /fixtures/simple_view.json?_extra=is_view``
.. code-block:: json
true
``private``
Whether this table is private to the current actor
``GET /fixtures/facetable.json?_extra=private``
.. code-block:: json
false
``expandable_columns``
Foreign key columns that can be expanded with labels
``GET /fixtures/facetable.json?_extra=expandable_columns``
.. code-block:: json
[
[
{
"column": "_city_id",
"other_table": "facet_cities",
"other_column": "id"
},
"name"
]
]
``form_hidden_args``
Hidden form arguments used by the HTML table interface
``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args``
.. code-block:: json
[
[
"_facet",
"state"
],
[
"_size",
"1"
],
[
"_extra",
"form_hidden_args"
]
]
.. [[[end]]]

View file

@ -1,12 +1,20 @@
import asyncio
import json
import pathlib
import tempfile
import textwrap
def table_extras(cog):
from datasette.extras import ExtraScope
from datasette.views.table_extras import table_extra_registry
cog.out("\n.. list-table::\n")
cog.out(" :header-rows: 1\n\n")
cog.out(" * - Extra\n")
cog.out(" - Description\n")
for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE):
classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE)
live_examples = asyncio.run(_fetch_live_examples(classes))
cog.out("\n")
for cls in classes:
example = cls.example
description = cls.description or ""
notes = []
if cls.expensive:
@ -15,6 +23,46 @@ def table_extras(cog):
notes.append(cls.docs_note)
if notes:
description = "{} ({})".format(description, " ".join(notes)).strip()
cog.out(" * - ``{}``\n".format(cls.key()))
cog.out(" - {}\n".format(description))
cog.out("\n")
cog.out("``{}``\n".format(cls.key()))
cog.out(" {}\n\n".format(description))
if example is None:
continue
if example.path:
value = live_examples[(example.path, example.key or cls.key())]
cog.out(" ``GET {}``\n\n".format(example.path))
else:
value = example.value
if example.note:
cog.out(" {}\n\n".format(example.note))
cog.out(" .. code-block:: json\n\n")
cog.out(textwrap.indent(json.dumps(value, indent=2), " "))
cog.out("\n\n")
async def _fetch_live_examples(classes):
from datasette.app import Datasette
from datasette.fixtures import write_fixture_database
examples = {}
with tempfile.TemporaryDirectory() as tmpdir:
db_path = pathlib.Path(tmpdir) / "fixtures.db"
write_fixture_database(db_path)
datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1})
try:
for cls in classes:
example = cls.example
if example is None or not example.path:
continue
key = example.key or cls.key()
response = await datasette.client.get(example.path)
assert response.status_code == 200, example.path
data = response.json()
assert key in data, "{} missing from {}".format(key, example.path)
examples[(example.path, key)] = data[key]
finally:
for db in datasette.databases.values():
if not db.is_memory:
db.close()
return examples

View file

@ -112,6 +112,16 @@ def test_table_filters_are_documented(documented_table_filters, subtests):
assert f.key in documented_table_filters
def test_table_extra_examples_are_documented():
from datasette.views.table_extras import CountExtra
assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count"
content = (docs_path / "json_api.rst").read_text()
section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0]
assert "GET /fixtures/facetable.json?_extra=count" in section
assert ".. code-block:: json" in section
@pytest.fixture(scope="session")
def documented_labels():
labels = set()